<commit_before>%% Copyright 2016, <NAME>
%%
%% Licensed under the Apache License, Version 2.0 (the "License"); you may not
%% use this file except in compliance with the License. You may obtain a copy of
%% the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
%% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
%% License for the specific language governing permissions and limitations under
%% the License.

%% Created by benoitc on 03/09/16.

-module(barrel_rocksdb_store).
-author("<NAME>").

-behaviour(barrel_store).

-include("barrel.hrl").

%% API
-export([
  init/2,
  open_db/3,
  clean_db/3,
  all_dbs/1,
  get_doc_info/3, get_doc_info/4,
  write_doc/6,
  get_doc/7,
  fold_by_id/5,
  changes_since/6,
  last_update_seq/2,
  write_system_doc/4,
  read_system_doc/3,
  delete_system_doc/3
]).

-define(VERSION, 1).

init(_Name, #{ store_backend := Backend }) ->
  Db = barrel_rocksdb_backend:get_db(Backend),
  {ok, #{db => Db }}.

open_db(#{ db := Db }, Name, Options) ->
  CreateIfMissing = proplists:get_value(create_if_missing, Options, false),
  DbKey = << 0, 0, 0, 100, (barrel_lib:to_binary(Name))/binary >>,
  case erocksdb:get(Db, DbKey, []) of
    {ok, DbId} ->
      UpdateSeq = get_update_seq(Db, DbId),
      {ok, {DbId, UpdateSeq}};
    not_found when CreateIfMissing /= false  ->
      DbId = barrel_lib:uniqid(),
      Batch =  [
                 {put, << 0, 0, 0, 0 >>, integer_to_binary(?VERSION)},
                 {put, DbKey, DbId},
                 {put, meta_key(DbId, 0), integer_to_binary(0)}
      ],
      ok = erocksdb:write(Db,Batch, [{sync, true}]),
      {ok, {DbId, 0}};
    not_found ->
      {error, not_found};
    Error ->
      Error
  end.

clean_db(Name, DbId, #{db := Db}) ->
  DbKey = << 0, 0, 0, 100, (barrel_lib:to_binary(Name))/binary >>,
  ok = erocksdb:delete(Db, DbKey, [{sync, true}]),
  _ = spawn(fun() ->
          fold_prefix(Db, DbId, fun clean_db_fun/3, {Db, []})
        end),
  ok.

clean_db_fun(K, _V, {Db, Acc}) ->
  Acc2 = [{delete, K} | Acc],
  Sz = length(Acc2),
  if
    Sz >= 500 ->
      _ = erocksdb:write(Db, Acc2, []),<commit_msg>append the name to the database id<commit_after>%% Copyright 2016, <NAME>
%%
%% Licensed under the Apache License, Version 2.0 (the "License"); you may not
%% use this file except in compliance with the License. You may obtain a copy of
%% the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
%% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
%% License for the specific language governing permissions and limitations under
%% the License.

%% Created by benoitc on 03/09/16.

-module(barrel_rocksdb_store).
-author("<NAME>").

-behaviour(barrel_store).

-include("barrel.hrl").

%% API
-export([
  init/2,
  open_db/3,
  clean_db/3,
  all_dbs/1,
  get_doc_info/3, get_doc_info/4,
  write_doc/6,
  get_doc/7,
  fold_by_id/5,
  changes_since/6,
  last_update_seq/2,
  write_system_doc/4,
  read_system_doc/3,
  delete_system_doc/3
]).

-define(VERSION, 1).

init(_Name, #{ store_backend := Backend }) ->
  Db = barrel_rocksdb_backend:get_db(Backend),
  {ok, #{db => Db }}.

open_db(#{ db := Db }, Name, Options) ->
  CreateIfMissing = proplists:get_value(create_if_missing, Options, false),
  DbKey = << 0, 0, 0, 100, (barrel_lib:to_binary(Name))/binary >>,
  case erocksdb:get(Db, DbKey, []) of
    {ok, DbId} ->
      UpdateSeq = get_update_seq(Db, DbId),
      {ok, {DbId, UpdateSeq}};
    not_found when CreateIfMissing /= false  ->
      DbId = << (barrel_lib:uniqid())/binary, "-", Name/binary >>,
      Batch =  [
                 {put, << 0, 0, 0, 0 >>, integer_to_binary(?VERSION)},
                 {put, DbKey, DbId},
                 {put, meta_key(DbId, 0), integer_to_binary(0)}
      ],
      ok = erocksdb:write(Db,Batch, [{sync, true}]),
      {ok, {DbId, 0}};
    not_found ->
      {error, not_found};
    Error ->
      Error
  end.

clean_db(Name, DbId, #{db := Db}) ->
  DbKey = << 0, 0, 0, 100, (barrel_lib:to_binary(Name))/binary >>,
  ok = erocksdb:delete(Db, DbKey, [{sync, true}]),
  _ = spawn(fun() ->
          fold_prefix(Db, DbId, fun clean_db_fun/3, {Db, []})
        end),
  ok.

clean_db_fun(K, _V, {Db, Acc}) ->
  Acc2 = [{delete, K} | Acc],
  Sz = length(Acc2),
  if
    Sz >= 500 ->
      _ = erocksdb:write(Db, Acc2, []),
{'fraction_non_alphanumeric': 0.16178371391641533, 'fraction_numerical': 0.023696682464454975, 'mean_word_length': 3.16786355475763, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3009181', 'n_tokens_mistral': 1802, 'n_tokens_neox': 1699, 'n_words': 437}