<commit_before>#!/usr/bin/env escript

-mode(compile).

-include("include/pulsedb.hrl").


main([]) ->
  code:add_pathz("ebin"),
  os:cmd("rm -rf benchdb"),
  SourceCount = 10,
  MinuteCount = 1440,
  T1 = erlang:now(),
  {ok, DB1} = pulsedb:open("benchdb"),
  Sources = [ <<"source", (integer_to_binary(I))/binary>> || I <- lists:seq(1,SourceCount)],

  DB2 = write_minutes(Sources, MinuteCount, DB1),
  pulsedb:close(DB2),
  T2 = erlang:now(),

  Count = SourceCount*MinuteCount*60*2,
  Time = timer:now_diff(T2,T1),

  Size_ = os:cmd("du -k -d 0 benchdb"),
  {match, [Size__]} = re:run(Size_, "(\\d+)", [{capture,all_but_first,list}]),
  Size = list_to_integer(Size__) / 1024,
  RawSize = Count*8 / (1024*1024),
  Ratio = RawSize / Size,
  io:format("written ~B data points in ~B seconds: ~B us per point. on disk: ~.1f MB, raw: ~.1f MB, ratio: ~.1f \n", 
    [Count, Time div 1000000, Time div Count, Size, RawSize, Ratio]),

  T4 = erlang:now(),
  {ok, DB3} = pulsedb:open("benchdb"),
  {ok, Ticks, DB4} = pulsedb:read([{name,<<"source2">>},{from,"1970-01-01"},{to,"1970-01-02"}], DB3),
  pulsedb:close(DB4),
  T5 = erlang:now(),
  Time2 = timer:now_diff(T5,T4),
  io:format("Extracted ~B ticks during ~B ms, ~B us per tick\n", [length(Ticks), Time2 div 1000, Time2 div length(Ticks)]),<commit_msg>make 100 sources by default<commit_after>#!/usr/bin/env escript

-mode(compile).

-include("include/pulsedb.hrl").


main([]) ->
  code:add_pathz("ebin"),
  os:cmd("rm -rf benchdb"),
  SourceCount = 100,
  MinuteCount = 1440,
  T1 = erlang:now(),
  {ok, DB1} = pulsedb:open("benchdb"),
  Sources = [ <<"source", (integer_to_binary(I))/binary>> || I <- lists:seq(1,SourceCount)],

  DB2 = write_minutes(Sources, MinuteCount, DB1),
  pulsedb:close(DB2),
  T2 = erlang:now(),

  Count = SourceCount*MinuteCount*60*2,
  Time = timer:now_diff(T2,T1),

  Size_ = os:cmd("du -k -d 0 benchdb"),
  {match, [Size__]} = re:run(Size_, "(\\d+)", [{capture,all_but_first,list}]),
  Size = list_to_integer(Size__) / 1024,
  RawSize = Count*8 / (1024*1024),
  Ratio = RawSize / Size,
  io:format("written ~B data points in ~B seconds: ~B us per point. on disk: ~.1f MB, raw: ~.1f MB, ratio: ~.1f \n", 
    [Count, Time div 1000000, Time div Count, Size, RawSize, Ratio]),

  T4 = erlang:now(),
  {ok, DB3} = pulsedb:open("benchdb"),
  {ok, Ticks, DB4} = pulsedb:read([{name,<<"source2">>},{from,"1970-01-01"},{to,"1970-01-02"}], DB3),
  pulsedb:close(DB4),
  T5 = erlang:now(),
  Time2 = timer:now_diff(T5,T4),
  io:format("Extracted ~B ticks during ~B ms, ~B us per tick\n", [length(Ticks), Time2 div 1000, Time2 div length(Ticks)]),
  os:cmd("rm -rf benchdb"),
{'fraction_non_alphanumeric': 0.21026802567006417, 'fraction_numerical': 0.05738014345035863, 'mean_word_length': 4.509355509355509, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6214913', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1078, 'n_words': 240}