<commit_before>%%
%%   Copyright 2012 <NAME>, All Rights Reserved
%%   Copyright 2012 <NAME>, All Rights Reserved
%%
%%   Licensed under the Apache License, Version 2.0 (the "License");
%%   you may not use this file except in compliance with the License.
%%   You may obtain a copy of the License at
%%
%%       http://www.apache.org/licenses/LICENSE-2.0
%%
%%   Unless required by applicable law or agreed to in writing, software
%%   distributed under the License is distributed on an "AS IS" BASIS,
%%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%%   See the License for the specific language governing permissions and
%%   limitations under the License.
%%
%%  @description
%%     http stream-oriented encoder / decoder.
%%  
%%  @todo
%%     * optimize recbuf binary, append is expensive, make lists / queue model
%%     * limit size recbuf
-module(htstream).
-include("htstream.hrl").

-export([
   new/0
  ,new/1
  ,state/1
  ,version/1
  ,http/1
  ,packets/1
  ,octets/1
  ,decode/1
  ,decode/2
  ,encode/1
  ,encode/2 
]).

-export_type([
   method/0, url/0, header/0, request/0, payload/0, http/0
]).

%%
%% public types 
-type(method()  :: atom()).                        % request method
-type(url()     :: binary()).                      % request url 
-type(header()  :: {atom() | binary(), binary() | integer}).  % http header
-type(request() :: {method(), url(), [header()]}). % http request
-type(response():: {integer(), binary(), [header()]}). 
-type(payload() :: binary()).                      % http payload
-type(http()    :: #http{}).                       % http parser state

-define(VERSION,   {1, 1}).

%%
%% create new http stream parser
-spec(new/0 :: () -> http()).
-spec(new/1 :: (any()) -> http()).

new()  ->
   new(?VERSION).

new({_, _}=Vsn) ->
   #http{is=idle, version=Vsn};

new(#http{recbuf=Buf, version=Vsn}) ->
   #http{is=idle, version=Vsn, recbuf=Buf}.

%%
%% check parser state
%%   idle    - 
%%   header  - handling headers
%%   payload - handling payload
%%   eoh     - end of headers
%%   eof     - end of message  
%%   upgrade - upgrade is requested
-spec(state/1 :: (#http{}) -> idle | header | payload | eof | upgrade).

state(#http{is=idle})    -> idle;
state(#http{is=header})  -> header;
state(#http{is=entity})  -> payload;
state(#http{is=chunk_head}) -> payload;
state(#http{is=chunk_data}) -> payload;
state(#http{is=chunk_tail}) -> payload;
state(#http{is=eoh})     -> eoh;
state(#http{is=eof})     -> eof;
state(#http{is=upgrade}) -> upgrade.

%%
%% return version of http stream
-spec(version/1 :: (#http{}) -> {integer(), integer()} | undefined).

version(#http{version=X}) ->
   X.

%%
%% return http request / response
-spec(http/1 :: (#http{}) -> {request | response, any()} | undefined).

http(#http{type=request,  htline={Method, Path}, headers=Head}) ->
   {request,  {Method, Path, Head}};

http(#http{type=response, htline={Status,  Msg}, headers=Head}) ->
   {response, {Status, Msg,  Head}};

http(#http{}) ->
   undefined.

%%
%% return number of processed packets
-spec(packets/1 :: (#http{}) -> integer()).

packets(#http{packets=X}) ->
   X.

%%
%% return number of processed octets
-spec(octets/1 :: (#http{}) -> integer()).

octets(#http{octets=X}) ->
   X.

%%
%% decodes http stream 
%% returns parsed value and new parser state
-spec(decode/2 :: (binary(), #http{}) -> {iolist() | request() | response(), #http{}}).

decode(#http{}=S) ->
   decode(<<>>, S);
decode(Msg) ->
   decode(Msg, new()).

decode(Msg, #http{recbuf = <<>>}=S) ->
   decode(Msg, [], 
      S#http{
         packets = S#http.packets + 1
        ,octets  = S#http.octets  + erlang:iolist_size(Msg)
      });
decode(Msg, S) ->
   decode(iolist_to_binary([S#http.recbuf, Msg]), [], 
      S#http{
         recbuf = <<>>
        ,packets = S#http.packets + 1
        ,octets  = S#http.octets  + erlang:iolist_size(Msg)
      }).

%%
%% encode http stream
%% returns produces http message and new parser state
-spec(encode/2 :: (iolist() | request() | response(), #http{}) -> {binary(), #http{}}).

encode(Msg) ->
   encode(Msg, new()).
encode(Msg, #http{}=S) ->
   encode(Msg, [], S).



%%%------------------------------------------------------------------
%%%
%%% decoder
%%%
%%%------------------------------------------------------------------

%% decode http request
decode(Pckt, _Acc, #http{is=idle}=S) ->
   decode_http(erlang:decode_packet(http_bin, Pckt, []), Pckt, S);

%% decode http headers
decode(Pckt, _Acc, #http{is=header}=S) ->
   decode_header(erlang:decode_packet(httph_bin, Pckt, []), Pckt, S);

%% decode entity payload (end-of-header / entity first byte)
decode(Pckt, Acc, #http{is=eoh, length=chunked}=S) ->
   decode(Pckt, Acc, S#http{is=chunk_head, length=0});
decode(Pckt, Acc, #http{is=eoh}=S) ->
   decode(Pckt, Acc, S#http{is=entity});

%% decode entity payload
decode(Pckt, Acc, #http{is=entity, length=Len}=S)
 when is_integer(Len), size(Pckt) < Len ->
   {lists:reverse([Pckt  | Acc]), S#http{length=Len - size(Pckt)}};

decode(Pckt, Acc, #http{is=entity, length=Len}=S)
 when is_integer(Len) ->
   <<Chunk:Len/binary, Rest/binary>> = Pckt,
   {lists:reverse([Chunk | Acc]), S#http{is=eof, length=0, recbuf=Rest}};

decode(Pckt, Acc, #http{is=entity, length=inf}=S) ->
   % message length is determined by closed connection
   {lists:reverse([Pckt  | Acc]), S};

%% decode chunked payload 
decode(Pckt, Acc, #http{is=chunk_head, length=0}=S) ->
   decode_chunk_head(binary:split(Pckt, <<"\r\n">>), Pckt, Acc, S);

decode(Pckt, Acc, #http{is=chunk_data, length=Len}=S)
 when size(Pckt) < Len ->
   {lists:reverse([Pckt  | Acc]), S#http{length=Len - size(Pckt)}};

decode(Pckt, Acc, #http{is=chunk_data, length=Len}=S) ->
   <<Chunk:Len/binary, Rest/binary>> = Pckt,
   decode(Rest, [Chunk | Acc], S#http{is=chunk_tail, length=0});

decode(Pckt, Acc, #http{is=chunk_tail}=S) ->
   decode_chunk_tail(binary:split(Pckt, <<"\r\n">>), Pckt, Acc, S);  

decode(Pckt, Acc, #http{is=eof}=State) ->
   {lists:reverse(Acc), State#http{recbuf=Pckt}}.


%% attempt to parse http request/response line
decode_http({more, _}, Pckt, S) ->
   {[], S#http{recbuf=Pckt}};
decode_http({error, _}, _Pckt, _S) ->
   exit(badarg);
decode_http({ok, {http_error, _}, _}, _Pckt, _S) ->
   exit(badarg);
decode_http({ok, {http_request, Mthd, Url, Vsn}, Rest}, _Pckt, S) ->
   decode(Rest, [],
      S#http{
         is      = header,
         type    = request,
         htline  = {decode_method(Mthd), decode_url(Url)},
         version = Vsn   
      }
   );
decode_http({ok, {http_response, Vsn, Status, Msg}, Rest}, _Pckt, S) ->
   decode(Rest, [],
      S#http{
         is      = header,
         type    = response,
         htline  = {Status, Msg},
         version = Vsn
      }
   ).

%% attempt to decode method
decode_method(Mthd) ->
   Mthd.

%% attempt to decode url
decode_url({abs_path, Url}) ->  
   Url;
decode_url({absoluteURI, Scheme, Host, undefined, Path}) ->
   <<(atom_to_binary(Scheme, utf8))/binary, $:, $/, $/, 
      Host/binary, Path/binary>>;
decode_url({absoluteURI, Scheme, Host, Port, Path}) ->
   <<(atom_to_binary(Scheme, utf8))/binary, $:, $/, $/, 
     Host/binary, $:, (list_to_binary(integer_to_list(Port)))/binary, Path/binary>>;
decode_url('*') ->
   <<$*>>.

%% attempt to parse HTTP header
decode_header({more, _}, Pckt, S) ->
   {[], S#http{recbuf=Pckt}};
decode_header({error, _}, _Pckt, _S) ->
   exit(badarg);
decode_header({ok, {http_error, _}, _}, _Pckt, _S) ->
   exit(badarg);
decode_header({ok, {http_header, _I, Head, _R, Val}, Rest}, _Pckt, S) -> 
   decode(Rest, [], S#http{headers=[decode_header_value(Head, Val)|S#http.headers]});
decode_header({ok, http_eoh, Rest}, _Pckt, State) ->
   {Mthd, Url} = State#http.htline,
   Head        = lists:reverse(State#http.headers),
   {{Mthd, Url, Head}, decode_check_payload(State#http{headers=Head, recbuf=Rest})}.


%% parse header value
decode_header_value('Content-Length', Val) ->
   {'Content-Length', list_to_integer(binary_to_list(Val))};
decode_header_value('Transfer-Length', Val) ->
   {'Transfer-Length', list_to_integer(binary_to_list(Val))};
decode_header_value('Content-Type', Val) ->
   {'Content-Type', decode_mime_type(Val)};
decode_header_value('Accept', Val) ->
   {'Accept', [decode_mime_type(X) || X <- binary:split(Val, <<$,>>, [trim, global])]};
decode_header_value(Head, Val) ->
   {Head, Val}.

%% parse chunk header
decode_chunk_head([_], Pckt, Acc, S) ->
   {lists:reverse(Acc), S#http{recbuf=Pckt}};
decode_chunk_head([Head, Pckt], _Pckt, Acc, S) ->
   [Len |_] = binary:split(Head, [<<" ">>, <<";">>]),
   case list_to_integer(binary_to_list(Len), 16) of
      0   ->
         %decode(Pckt, Acc, S#http{is=chunk_tail, length=0});
         % TODO: decoder assumes that 0\r\n\r\n is arrived in single packet
         <<_:2/binary, Rest/binary>> = Pckt,
         {lists:reverse(Acc), S#http{is=eof, recbuf=Rest}}; 
      Val ->
         decode(Pckt, Acc, S#http{is=chunk_data, length=Val}) 
   end.

%% parse chunk tail
decode_chunk_tail([_], Pckt, Acc, S) ->
   {lists:reverse(Acc), S#http{recbuf=Pckt}};
decode_chunk_tail([_, Pckt], _Pckt, Acc, S) ->
   decode(Pckt, Acc, S#http{is=chunk_head, length=0}).


%%
%% decode check message payload
%% see http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4
%%
%% 1. Any response message which "MUST NOT" include a message-body 
%%    (such as the 1xx, 204, and 304 responses and any response to 
%%    a HEAD request) is always terminated by the first empty line 
%%    after the header fields... (Not implemented)
%%
%% 2. If a Transfer-Encoding header field (section 14.41) is present 
%%    and has any value other than "identity", then the transfer-length 
%%    is defined by use of the "chunked" transfer-coding (section 3.6), 
%%    unless the message is terminated by closing the connection.
%%
%% 3. If a Content-Length header field (section 14.13) is present, 
%%    its decimal value in OCTETs represents both the entity-length 
%%    and the transfer-length. The Content-Length header field MUST NOT 
%%    be sent if these two lengths are different
%%
%% 4. If the message uses the media type "multipart/byteranges"... 
%%    (Not implemented)
%%
%% 5. By the server closing the connection. 
decode_check_payload(#http{htline={'GET',  _}}=State) ->
   case lists:keyfind('Connection', 1, State#http.headers) of
      {_, <<"Upgrade">>} ->
         State#http{is=upgrade};
      _ ->
         State#http{is=eof}
   end;
decode_check_payload(#http{htline={'HEAD', _}}=S) ->
   S#http{is=eof};
decode_check_payload(#http{htline={'DELETE', _}}=S) ->
   S#http{is=eof};
decode_check_payload(S) ->
   element(2, alt(S, [
      fun is_payload_chunked/1,
      fun is_payload_entity/1,
      fun is_payload_eof/1
   ])).

%%
%% TODO: support q-values
decode_mime_type(Val) ->
   [Mime | _QVal] = binary:split(Val, <<$;>>, []),
   case binary:split(Mime, <<$/>>, []) of
      [<<$*>>, <<$*>>] -> {'*',  '*'};
      [Type,   <<$*>>] -> {Type, '*'};
      [<<$*>>,SubType] -> {'*', SubType};
      [Type,  SubType] -> {Type,SubType};
      [Type]           -> {Type, '*'}
   end.


%%%------------------------------------------------------------------
%%%
%%% encoder
%%%
%%%------------------------------------------------------------------

%%
%%
encode(eof, _Acc, State) ->
   {[], State};
encode(Msg, _Acc, #http{is=idle}=S)   ->
   encode_http(erlang:element(1, Msg), Msg, S);
<commit_msg>fix: handle eof properly if encoder is idle<commit_after>%%
%%   Copyright 2012 <NAME>, All Rights Reserved
%%   Copyright 2012 <NAME>, All Rights Reserved
%%
%%   Licensed under the Apache License, Version 2.0 (the "License");
%%   you may not use this file except in compliance with the License.
%%   You may obtain a copy of the License at
%%
%%       http://www.apache.org/licenses/LICENSE-2.0
%%
%%   Unless required by applicable law or agreed to in writing, software
%%   distributed under the License is distributed on an "AS IS" BASIS,
%%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%%   See the License for the specific language governing permissions and
%%   limitations under the License.
%%
%%  @description
%%     http stream-oriented encoder / decoder.
%%  
%%  @todo
%%     * optimize recbuf binary, append is expensive, make lists / queue model
%%     * limit size recbuf
-module(htstream).
-include("htstream.hrl").

-export([
   new/0
  ,new/1
  ,state/1
  ,version/1
  ,http/1
  ,packets/1
  ,octets/1
  ,decode/1
  ,decode/2
  ,encode/1
  ,encode/2 
]).

-export_type([
   method/0, url/0, header/0, request/0, payload/0, http/0
]).

%%
%% public types 
-type(method()  :: atom()).                        % request method
-type(url()     :: binary()).                      % request url 
-type(header()  :: {atom() | binary(), binary() | integer}).  % http header
-type(request() :: {method(), url(), [header()]}). % http request
-type(response():: {integer(), binary(), [header()]}). 
-type(payload() :: binary()).                      % http payload
-type(http()    :: #http{}).                       % http parser state

-define(VERSION,   {1, 1}).

%%
%% create new http stream parser
-spec(new/0 :: () -> http()).
-spec(new/1 :: (any()) -> http()).

new()  ->
   new(?VERSION).

new({_, _}=Vsn) ->
   #http{is=idle, version=Vsn};

new(#http{recbuf=Buf, version=Vsn}) ->
   #http{is=idle, version=Vsn, recbuf=Buf}.

%%
%% check parser state
%%   idle    - 
%%   header  - handling headers
%%   payload - handling payload
%%   eoh     - end of headers
%%   eof     - end of message  
%%   upgrade - upgrade is requested
-spec(state/1 :: (#http{}) -> idle | header | payload | eof | upgrade).

state(#http{is=idle})    -> idle;
state(#http{is=header})  -> header;
state(#http{is=entity})  -> payload;
state(#http{is=chunk_head}) -> payload;
state(#http{is=chunk_data}) -> payload;
state(#http{is=chunk_tail}) -> payload;
state(#http{is=eoh})     -> eoh;
state(#http{is=eof})     -> eof;
state(#http{is=upgrade}) -> upgrade.

%%
%% return version of http stream
-spec(version/1 :: (#http{}) -> {integer(), integer()} | undefined).

version(#http{version=X}) ->
   X.

%%
%% return http request / response
-spec(http/1 :: (#http{}) -> {request | response, any()} | undefined).

http(#http{type=request,  htline={Method, Path}, headers=Head}) ->
   {request,  {Method, Path, Head}};

http(#http{type=response, htline={Status,  Msg}, headers=Head}) ->
   {response, {Status, Msg,  Head}};

http(#http{}) ->
   undefined.

%%
%% return number of processed packets
-spec(packets/1 :: (#http{}) -> integer()).

packets(#http{packets=X}) ->
   X.

%%
%% return number of processed octets
-spec(octets/1 :: (#http{}) -> integer()).

octets(#http{octets=X}) ->
   X.

%%
%% decodes http stream 
%% returns parsed value and new parser state
-spec(decode/2 :: (binary(), #http{}) -> {iolist() | request() | response(), #http{}}).

decode(#http{}=S) ->
   decode(<<>>, S);
decode(Msg) ->
   decode(Msg, new()).

decode(Msg, #http{recbuf = <<>>}=S) ->
   decode(Msg, [], 
      S#http{
         packets = S#http.packets + 1
        ,octets  = S#http.octets  + erlang:iolist_size(Msg)
      });
decode(Msg, S) ->
   decode(iolist_to_binary([S#http.recbuf, Msg]), [], 
      S#http{
         recbuf = <<>>
        ,packets = S#http.packets + 1
        ,octets  = S#http.octets  + erlang:iolist_size(Msg)
      }).

%%
%% encode http stream
%% returns produces http message and new parser state
-spec(encode/2 :: (iolist() | request() | response(), #http{}) -> {binary(), #http{}}).

encode(Msg) ->
   encode(Msg, new()).
encode(Msg, #http{}=S) ->
   encode(Msg, [], S).



%%%------------------------------------------------------------------
%%%
%%% decoder
%%%
%%%------------------------------------------------------------------

%% decode http request
decode(Pckt, _Acc, #http{is=idle}=S) ->
   decode_http(erlang:decode_packet(http_bin, Pckt, []), Pckt, S);

%% decode http headers
decode(Pckt, _Acc, #http{is=header}=S) ->
   decode_header(erlang:decode_packet(httph_bin, Pckt, []), Pckt, S);

%% decode entity payload (end-of-header / entity first byte)
decode(Pckt, Acc, #http{is=eoh, length=chunked}=S) ->
   decode(Pckt, Acc, S#http{is=chunk_head, length=0});
decode(Pckt, Acc, #http{is=eoh}=S) ->
   decode(Pckt, Acc, S#http{is=entity});

%% decode entity payload
decode(Pckt, Acc, #http{is=entity, length=Len}=S)
 when is_integer(Len), size(Pckt) < Len ->
   {lists:reverse([Pckt  | Acc]), S#http{length=Len - size(Pckt)}};

decode(Pckt, Acc, #http{is=entity, length=Len}=S)
 when is_integer(Len) ->
   <<Chunk:Len/binary, Rest/binary>> = Pckt,
   {lists:reverse([Chunk | Acc]), S#http{is=eof, length=0, recbuf=Rest}};

decode(Pckt, Acc, #http{is=entity, length=inf}=S) ->
   % message length is determined by closed connection
   {lists:reverse([Pckt  | Acc]), S};

%% decode chunked payload 
decode(Pckt, Acc, #http{is=chunk_head, length=0}=S) ->
   decode_chunk_head(binary:split(Pckt, <<"\r\n">>), Pckt, Acc, S);

decode(Pckt, Acc, #http{is=chunk_data, length=Len}=S)
 when size(Pckt) < Len ->
   {lists:reverse([Pckt  | Acc]), S#http{length=Len - size(Pckt)}};

decode(Pckt, Acc, #http{is=chunk_data, length=Len}=S) ->
   <<Chunk:Len/binary, Rest/binary>> = Pckt,
   decode(Rest, [Chunk | Acc], S#http{is=chunk_tail, length=0});

decode(Pckt, Acc, #http{is=chunk_tail}=S) ->
   decode_chunk_tail(binary:split(Pckt, <<"\r\n">>), Pckt, Acc, S);  

decode(Pckt, Acc, #http{is=eof}=State) ->
   {lists:reverse(Acc), State#http{recbuf=Pckt}}.


%% attempt to parse http request/response line
decode_http({more, _}, Pckt, S) ->
   {[], S#http{recbuf=Pckt}};
decode_http({error, _}, _Pckt, _S) ->
   exit(badarg);
decode_http({ok, {http_error, _}, _}, _Pckt, _S) ->
   exit(badarg);
decode_http({ok, {http_request, Mthd, Url, Vsn}, Rest}, _Pckt, S) ->
   decode(Rest, [],
      S#http{
         is      = header,
         type    = request,
         htline  = {decode_method(Mthd), decode_url(Url)},
         version = Vsn   
      }
   );
decode_http({ok, {http_response, Vsn, Status, Msg}, Rest}, _Pckt, S) ->
   decode(Rest, [],
      S#http{
         is      = header,
         type    = response,
         htline  = {Status, Msg},
         version = Vsn
      }
   ).

%% attempt to decode method
decode_method(Mthd) ->
   Mthd.

%% attempt to decode url
decode_url({abs_path, Url}) ->  
   Url;
decode_url({absoluteURI, Scheme, Host, undefined, Path}) ->
   <<(atom_to_binary(Scheme, utf8))/binary, $:, $/, $/, 
      Host/binary, Path/binary>>;
decode_url({absoluteURI, Scheme, Host, Port, Path}) ->
   <<(atom_to_binary(Scheme, utf8))/binary, $:, $/, $/, 
     Host/binary, $:, (list_to_binary(integer_to_list(Port)))/binary, Path/binary>>;
decode_url('*') ->
   <<$*>>.

%% attempt to parse HTTP header
decode_header({more, _}, Pckt, S) ->
   {[], S#http{recbuf=Pckt}};
decode_header({error, _}, _Pckt, _S) ->
   exit(badarg);
decode_header({ok, {http_error, _}, _}, _Pckt, _S) ->
   exit(badarg);
decode_header({ok, {http_header, _I, Head, _R, Val}, Rest}, _Pckt, S) -> 
   decode(Rest, [], S#http{headers=[decode_header_value(Head, Val)|S#http.headers]});
decode_header({ok, http_eoh, Rest}, _Pckt, State) ->
   {Mthd, Url} = State#http.htline,
   Head        = lists:reverse(State#http.headers),
   {{Mthd, Url, Head}, decode_check_payload(State#http{headers=Head, recbuf=Rest})}.


%% parse header value
decode_header_value('Content-Length', Val) ->
   {'Content-Length', list_to_integer(binary_to_list(Val))};
decode_header_value('Transfer-Length', Val) ->
   {'Transfer-Length', list_to_integer(binary_to_list(Val))};
decode_header_value('Content-Type', Val) ->
   {'Content-Type', decode_mime_type(Val)};
decode_header_value('Accept', Val) ->
   {'Accept', [decode_mime_type(X) || X <- binary:split(Val, <<$,>>, [trim, global])]};
decode_header_value(Head, Val) ->
   {Head, Val}.

%% parse chunk header
decode_chunk_head([_], Pckt, Acc, S) ->
   {lists:reverse(Acc), S#http{recbuf=Pckt}};
decode_chunk_head([Head, Pckt], _Pckt, Acc, S) ->
   [Len |_] = binary:split(Head, [<<" ">>, <<";">>]),
   case list_to_integer(binary_to_list(Len), 16) of
      0   ->
         %decode(Pckt, Acc, S#http{is=chunk_tail, length=0});
         % TODO: decoder assumes that 0\r\n\r\n is arrived in single packet
         <<_:2/binary, Rest/binary>> = Pckt,
         {lists:reverse(Acc), S#http{is=eof, recbuf=Rest}}; 
      Val ->
         decode(Pckt, Acc, S#http{is=chunk_data, length=Val}) 
   end.

%% parse chunk tail
decode_chunk_tail([_], Pckt, Acc, S) ->
   {lists:reverse(Acc), S#http{recbuf=Pckt}};
decode_chunk_tail([_, Pckt], _Pckt, Acc, S) ->
   decode(Pckt, Acc, S#http{is=chunk_head, length=0}).


%%
%% decode check message payload
%% see http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4
%%
%% 1. Any response message which "MUST NOT" include a message-body 
%%    (such as the 1xx, 204, and 304 responses and any response to 
%%    a HEAD request) is always terminated by the first empty line 
%%    after the header fields... (Not implemented)
%%
%% 2. If a Transfer-Encoding header field (section 14.41) is present 
%%    and has any value other than "identity", then the transfer-length 
%%    is defined by use of the "chunked" transfer-coding (section 3.6), 
%%    unless the message is terminated by closing the connection.
%%
%% 3. If a Content-Length header field (section 14.13) is present, 
%%    its decimal value in OCTETs represents both the entity-length 
%%    and the transfer-length. The Content-Length header field MUST NOT 
%%    be sent if these two lengths are different
%%
%% 4. If the message uses the media type "multipart/byteranges"... 
%%    (Not implemented)
%%
%% 5. By the server closing the connection. 
decode_check_payload(#http{htline={'GET',  _}}=State) ->
   case lists:keyfind('Connection', 1, State#http.headers) of
      {_, <<"Upgrade">>} ->
         State#http{is=upgrade};
      _ ->
         State#http{is=eof}
   end;
decode_check_payload(#http{htline={'HEAD', _}}=S) ->
   S#http{is=eof};
decode_check_payload(#http{htline={'DELETE', _}}=S) ->
   S#http{is=eof};
decode_check_payload(S) ->
   element(2, alt(S, [
      fun is_payload_chunked/1,
      fun is_payload_entity/1,
      fun is_payload_eof/1
   ])).

%%
%% TODO: support q-values
decode_mime_type(Val) ->
   [Mime | _QVal] = binary:split(Val, <<$;>>, []),
   case binary:split(Mime, <<$/>>, []) of
      [<<$*>>, <<$*>>] -> {'*',  '*'};
      [Type,   <<$*>>] -> {Type, '*'};
      [<<$*>>,SubType] -> {'*', SubType};
      [Type,  SubType] -> {Type,SubType};
      [Type]           -> {Type, '*'}
   end.


%%%------------------------------------------------------------------
%%%
%%% encoder
%%%
%%%------------------------------------------------------------------

%%
%%
encode(eof, _Acc, #http{is=idle}=S) ->
   {[], S};
encode(Msg, _Acc, #http{is=idle}=S)   ->
   encode_http(erlang:element(1, Msg), Msg, S);

{'fraction_non_alphanumeric': 0.21577594842980966, 'fraction_numerical': 0.008188509952524065, 'mean_word_length': 3.2455621301775146, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 261, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '868360', 'n_tokens_mistral': 8358, 'n_tokens_neox': 8137, 'n_words': 1857}