<commit_before>-module(stockdb_filters).
-author('<NAME> <<EMAIL>>').

-include("../include/stockdb.hrl").
-include_lib("eunit/include/eunit.hrl").


-export([candle/2]).
% -export([average/2]).

-record(candle, {
  granulation,
  current_segment,
  open,
  high_ask,
  low_bid,
  close
}).

-define(GRANULATION, 30000).

candle(Event, Candle) when not is_record(Event, md) andalso Event =/= eof ->
  {[Event], Candle};

candle(Event, undefined) ->
  candle(Event, []);

candle(#md{} = MD, Opts) when is_list(Opts) ->
  Granulation = proplists:get_value(granulation, Opts, ?GRANULATION),
  candle(MD, #candle{
    granulation = Granulation,
    current_segment = -1});

candle(#md{timestamp = Timestamp} = MD, #candle{granulation = Granulation, current_segment = Seg} = Candle) 
  when Timestamp div Granulation > Seg ->
  {Events, Candle1} = flush_segment(Candle),
  {Events, Candle1#candle{
      current_segment = Timestamp div Granulation,
      open = MD,
      high_ask = MD,
      low_bid = MD,
      close = MD
    }};

candle(eof, #candle{} = Candle) ->
  flush_segment(Candle);

candle(#md{} = MD, #candle{} = Candle) ->
  {[], accumulate_md(MD, Candle)}.


flush_segment(#candle{open = Open, high_ask = HighAsk, low_bid = LowBid, close = Close} = Candle) ->
  Events = lists:usort([Open, HighAsk, LowBid, Close]),
  {Events, Candle#candle{open = undefined, high_ask = undefined, low_bid = undefined, close = undefined}}.


accumulate_md(#md{bid = [{Bid,_}|_], ask = [{Ask,_}|_]} = MD, #candle{high_ask = HighAsk, low_bid = LowBid} = Candle) ->
  HA1 = case HighAsk of
    undefined -> MD;
    #md{ask = [{Ask1,_}|_]} when Ask > Ask1 -> MD;
    _ -> HighAsk
  end,

  LB1 = case LowBid of
    undefined -> MD;
    #md{bid = [{Bid1,_}|_]} when Bid < Bid1 -> MD;
    _ -> LowBid
  end,
  
  Candle#candle{high_ask = HA1, low_bid = LB1, close = MD}.

<commit_msg>stockdb: candle filter: explicitly filter out undefined events<commit_after>-module(stockdb_filters).
-author('<NAME> <<EMAIL>>').

-include("../include/stockdb.hrl").
-include_lib("eunit/include/eunit.hrl").


-export([candle/2]).
% -export([average/2]).

-record(candle, {
  granulation,
  current_segment,
  open,
  high_ask,
  low_bid,
  close
}).

-define(GRANULATION, 30000).

candle(Event, Candle) when not is_record(Event, md) andalso Event =/= eof ->
  {[Event], Candle};

candle(Event, undefined) ->
  candle(Event, []);

candle(#md{} = MD, Opts) when is_list(Opts) ->
  Granulation = proplists:get_value(granulation, Opts, ?GRANULATION),
  candle(MD, #candle{
    granulation = Granulation,
    current_segment = -1});

candle(#md{timestamp = Timestamp} = MD, #candle{granulation = Granulation, current_segment = Seg} = Candle) 
  when Timestamp div Granulation > Seg ->
  {Events, Candle1} = flush_segment(Candle),
  {Events, Candle1#candle{
      current_segment = Timestamp div Granulation,
      open = MD,
      high_ask = MD,
      low_bid = MD,
      close = MD
    }};

candle(eof, #candle{} = Candle) ->
  flush_segment(Candle);

candle(#md{} = MD, #candle{} = Candle) ->
  {[], accumulate_md(MD, Candle)}.


flush_segment(#candle{open = Open, high_ask = HighAsk, low_bid = LowBid, close = Close} = Candle) ->
  Events = lists:usort([Open, HighAsk, LowBid, Close]),
  {Events -- [undefined], Candle#candle{open = undefined, high_ask = undefined, low_bid = undefined, close = undefined}}.


accumulate_md(#md{bid = [{Bid,_}|_], ask = [{Ask,_}|_]} = MD, #candle{high_ask = HighAsk, low_bid = LowBid} = Candle) ->
  HA1 = case HighAsk of
    undefined -> MD;
    #md{ask = [{Ask1,_}|_]} when Ask > Ask1 -> MD;
    _ -> HighAsk
  end,

  LB1 = case LowBid of
    undefined -> MD;
    #md{bid = [{Bid1,_}|_]} when Bid < Bid1 -> MD;
    _ -> LowBid
  end,
  
  Candle#candle{high_ask = HA1, low_bid = LB1, close = MD}.


{'fraction_non_alphanumeric': 0.17245005257623555, 'fraction_numerical': 0.00946372239747634, 'mean_word_length': 3.7443890274314215, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '203456', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1468, 'n_words': 312}