<commit_before>-module(boss_db_adapter_dynamodb).
-behaviour(boss_db_adapter).
-export([start/1, stop/0, init/1, terminate/1, find/2, find/7]).
-export([count/3, counter/2, incr/2, incr/3, delete/2, save_record/2]).
-export([push/2, pop/2]).

-define(LOG(Name, Value), io:format("DEBUG: ~s: ~p~n", [Name, Value])).

% Number of seconds between beginning of gregorian calendar and 1970
-define(GREGORIAN_SECONDS_1970, 62167219200).

start(_Options) ->
    application:start(ddb).

stop() ->
    ok.

init(Options) ->
	AccessKey 	= proplists:get_value(db_username, Options, os:getenv("AWS_ACCESS_KEY_ID")),
	SecretKey	= proplists:get_value(db_password, Options, os:getenv("AWS_SECRET_ACCESS_KEY")),
    Endpoint = proplists:get_value(db_host, Options, "dynamodb.us-east-1.amazonaws.com"),

	%% startup dependencies.  some of these may have already been started, but that's ok.
	inets:start(),
	ssl:start(),
	%%lager:start(),
	application:start(ibrowse),


	%% init initial credentials.  note that these will be refeshed automatically as needed
	ddb_iam:credentials(AccessKey, SecretKey),
	{'ok', Key, Secret, Token} = ddb_iam:token(129600),  %% 129600 is the lifetime duration for the token
	ddb:credentials(Key, Secret, Token, Endpoint),

	init_tables(),
    {ok, undefined}.

terminate(_Conn) ->
    ok.

find(_Conn, Id) when is_list(Id) ->
	[Type | _BossId] = string:tokens(Id, "-"),
	case ddb:get(list_to_binary(Type), ddb:key_value(list_to_binary(Id), 'string')) of
		{ok, Rec} ->
			create_from_ddbitem(Type, proplists:get_value(<<"Item">>, Rec));
		{error, Error} ->
			{error, Error};
		X ->
			{error, X}
	end.

find(_Conn, Type, Conditions, Max, Skip, Sort, SortOrder) when is_atom(Type), is_list(Conditions),
                                                              is_integer(Max) orelse Max =:= all, is_integer(Skip),
                                                              is_atom(Sort), is_atom(SortOrder) ->

	case boss_record_lib:ensure_loaded(Type) of
		true ->
			DDB_cond	= convert_conditions(Conditions),
			Items		= do_scan_loop(atom_to_binary(Type, latin1), DDB_cond, 'none'),
			Records		= create_from_ddbitem_list(Type, Items),

			case SortOrder of
				ascending -> Sorted = lists:sort(fun(A,B) -> A:Sort() =< B:Sort() end, Records);
				descending -> Sorted = lists:sort(fun(A,B) -> A:Sort() > B:Sort() end, Records);
				_ -> Sorted = Records
			end,

			Sorted;
		false ->
			{error, {module_not_loaded, Type}}
	end.


count(_Conn, _Type, _Conditions) ->
    1.

counter(_Conn, Id) when is_list(Id) ->
    1.

incr(Conn, Id) ->
    incr(Conn, Id, 1).

incr(Conn, Id, _Count) ->
    Res = {ok, ok},
    case Res of
        {ok, ok} -> counter(Conn, Id);
        {failure, Reason} -> {error, Reason};
        {connection_failure, Reason} -> {error, Reason}
    end.

delete(_Conn, Id) when is_list(Id) ->

    Res = {ok, ok},

    case Res of
        {ok, ok} -> ok;
        {failure, Reason} -> {error, Reason};
        {connection_failure, Reason} -> {error, Reason}
    end.


save_record(_Conn, Record) when is_tuple(Record) ->
	case Record:id() of
		id ->
			create_new(Record);
		_Existing ->
			update_existing(Record)
	end.


% These 2 functions are not part of the behaviour but are required for
% tests to pass
push(_Conn, _Depth) -> ok.

pop(_Conn, _Depth) -> ok.




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Internal functions
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%
%% setup the DynamoDB tables as needed
%%

init_tables() ->
	{ok, Tables} = ddb:tables(),
	Models = boss_files:model_list(cx),  %% TODO make this more generic
    [{loaded, LoadedModules}|_] = application:info(),
    Models = lists:concat( [ boss_files:model_list(ModelName) || {ModelName, _, _} <- LoadedModules ] ),
	BinModels = [ erlang:list_to_binary(X) || X <- Models],
	Create = BinModels -- Tables,
	init_models(Create).

init_models([]) ->
	ok;
init_models([Model | Tail]) ->
	init_table(Model),
	init_models(Tail).

init_table(Model) when is_binary(Model) ->
	%% it looks like we don't need to initialize the table with the field names for now
	%%Attrs = (boss_record_lib:dummy_record(Model)):attribute_names(),
	ddb:create_table(Model, ddb:key_type(<<"id">>, 'string'), 10, 10),
	init_meta_entry(Model).

init_meta_entry(Model) when is_binary(Model)->
	ddb:put(Model, [{<<"id">>, <<"metadata">>, 'string'},
					{<<"counter">>, <<"0">>, 'number'}]).

%%
%% Helper functions for creating records

create_new(Record) ->
	Model = element(1, Record),
	SavedRec = Record:set(id, get_new_unique_id(Model)),
	Attrs = SavedRec:attributes(),
	%% WEIRD: dynamodb does not allow empty strings as values, don't store them.<commit_msg>Add global/local config values for read/write capacity. <commit_after>-module(boss_db_adapter_dynamodb).
-behaviour(boss_db_adapter).
-export([start/1, stop/0, init/1, terminate/1, find/2, find/7]).
-export([count/3, counter/2, incr/2, incr/3, delete/2, save_record/2]).
-export([push/2, pop/2]).

-define(LOG(Name, Value), io:format("DEBUG: ~s: ~p~n", [Name, Value])).

% Number of seconds between beginning of gregorian calendar and 1970
-define(GREGORIAN_SECONDS_1970, 62167219200).

start(_Options) ->
    application:start(ddb).

stop() ->
    ok.

init(Options) ->
	AccessKey 	= proplists:get_value(db_username, Options, os:getenv("AWS_ACCESS_KEY_ID")),
	SecretKey	= proplists:get_value(db_password, Options, os:getenv("AWS_SECRET_ACCESS_KEY")),
    Endpoint = proplists:get_value(db_host, Options, "dynamodb.us-east-1.amazonaws.com"),

	%% startup dependencies.  some of these may have already been started, but that's ok.
	inets:start(),
	ssl:start(),
	%%lager:start(),
	application:start(ibrowse),


	%% init initial credentials.  note that these will be refeshed automatically as needed
	ddb_iam:credentials(AccessKey, SecretKey),
	{'ok', Key, Secret, Token} = ddb_iam:token(129600),  %% 129600 is the lifetime duration for the token
	ddb:credentials(Key, Secret, Token, Endpoint),

	init_tables(Options),
    {ok, undefined}.

terminate(_Conn) ->
    ok.

find(_Conn, Id) when is_list(Id) ->
	[Type | _BossId] = string:tokens(Id, "-"),
	case ddb:get(list_to_binary(Type), ddb:key_value(list_to_binary(Id), 'string')) of
		{ok, Rec} ->
			create_from_ddbitem(Type, proplists:get_value(<<"Item">>, Rec));
		{error, Error} ->
			{error, Error};
		X ->
			{error, X}
	end.

find(_Conn, Type, Conditions, Max, Skip, Sort, SortOrder) when is_atom(Type), is_list(Conditions),
                                                              is_integer(Max) orelse Max =:= all, is_integer(Skip),
                                                              is_atom(Sort), is_atom(SortOrder) ->

	case boss_record_lib:ensure_loaded(Type) of
		true ->
			DDB_cond	= convert_conditions(Conditions),
			Items		= do_scan_loop(atom_to_binary(Type, latin1), DDB_cond, 'none'),
			Records		= create_from_ddbitem_list(Type, Items),

			case SortOrder of
				ascending -> Sorted = lists:sort(fun(A,B) -> A:Sort() =< B:Sort() end, Records);
				descending -> Sorted = lists:sort(fun(A,B) -> A:Sort() > B:Sort() end, Records);
				_ -> Sorted = Records
			end,

			Sorted;
		false ->
			{error, {module_not_loaded, Type}}
	end.


count(_Conn, _Type, _Conditions) ->
    1.

counter(_Conn, Id) when is_list(Id) ->
    1.

incr(Conn, Id) ->
    incr(Conn, Id, 1).

incr(Conn, Id, _Count) ->
    Res = {ok, ok},
    case Res of
        {ok, ok} -> counter(Conn, Id);
        {failure, Reason} -> {error, Reason};
        {connection_failure, Reason} -> {error, Reason}
    end.

delete(_Conn, Id) when is_list(Id) ->

    Res = {ok, ok},

    case Res of
        {ok, ok} -> ok;
        {failure, Reason} -> {error, Reason};
        {connection_failure, Reason} -> {error, Reason}
    end.


save_record(_Conn, Record) when is_tuple(Record) ->
	case Record:id() of
		id ->
			create_new(Record);
		_Existing ->
			update_existing(Record)
	end.


% These 2 functions are not part of the behaviour but are required for
% tests to pass
push(_Conn, _Depth) -> ok.

pop(_Conn, _Depth) -> ok.




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Internal functions
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%
%% setup the DynamoDB tables as needed
%%

init_tables(Options) ->
	{ok, Tables} = ddb:tables(),
    [{loaded, LoadedModules}|_] = application:info(),
    Models = lists:concat( [ boss_files:model_list(ModelName) || {ModelName, _, _} <- LoadedModules ] ),
	BinModels = [ erlang:list_to_binary(X) || X <- Models],
	Create = BinModels -- Tables,
	init_models(Create, Options).

init_models([], _Options) ->
	ok;
init_models([Model | Tail], Options) ->
	init_table(Model, Options),
	init_models(Tail, Options).

init_table(Model, Options) when is_binary(Model) ->
	%% it looks like we don't need to initialize the table with the field names for now
    ModelAtom = erlang:binary_to_atom(Model, latin1),
    Dummy = boss_record_lib:dummy_record(ModelAtom),

    Settings =
        case erlang:function_exported(ModelAtom, dynamodb_settings, 1) of
            true ->
                Dummy:dynamodb_settings();
            _ ->
                %% Fallback on global configuration
                Options
        end,

    ReadCap = proplists:get_value(db_read_capacity, Settings, 2),
    WriteCap = proplists:get_value(db_write_capacity, Settings, 1),

    Keys =
        case proplists:get_value(range_key, Settings) of
            undefined ->
                ddb:key_type(<<"id">>, 'string');
            RangeKey ->
                ModelTypes = Dummy:attribute_types(),
                RealType = convert_type_to_ddb(proplists:get_value(RangeKey, ModelTypes)),
                ddb:key_type(<<"id">>, 'string', erlang:atom_to_binary(RangeKey, latin1), RealType)
        end,

	ddb:create_table(Model, Keys, ReadCap, WriteCap),
	init_meta_entry(Model).

%% Help function
convert_type_to_ddb(string) ->
    'string';
convert_type_to_ddb(float) ->
    'number';
convert_type_to_ddb(integer) ->
    'number';
convert_type_to_ddb(datetime) ->
    'number';
convert_type_to_ddb(timestamp) ->
    'number'.

init_meta_entry(Model) when is_binary(Model)->
	ddb:put(Model, [{<<"id">>, <<"metadata">>, 'string'},
					{<<"counter">>, <<"0">>, 'number'}]).

%%
%% Helper functions for creating records

create_new(Record) ->
	Model = element(1, Record),
	SavedRec = Record:set(id, get_new_unique_id(Model)),
	Attrs = SavedRec:attributes(),
	%% WEIRD: dynamodb does not allow empty strings as values, don't store them.
{'fraction_non_alphanumeric': 0.17603102534998108, 'fraction_numerical': 0.010877790389708665, 'mean_word_length': 3.6970235450910707, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 116, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '665071', 'n_tokens_mistral': 3856, 'n_tokens_neox': 3592, 'n_words': 827}