<commit_before>(*
This file is part of Arakoon, a distributed key-value store. Copyright
(C) 2010 Incubaid BVBA

Licensees holding a valid Incubaid license may use this file in
accordance with Incubaid's Arakoon commercial license agreement. For
more information on how to enter into this agreement, please contact
Incubaid (contact details can be found on www.arakoon.org/licensing).

Alternatively, this file may be redistributed and/or modified under
the terms of the GNU Affero General Public License version 3, as
published by the Free Software Foundation. Under this license, this
file is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.

See the GNU Affero General Public License for more details.
You should have received a copy of the
GNU Affero General Public License along with this program (file "COPYING").
If not, see <http://www.gnu.org/licenses/>.
*)

open Lwt

let head_name = Tlogcollection.head_name

let collapse_until tlog_dir head_name too_far_i =
  let tfs = Sn.string_of too_far_i in
  Lwt_log.debug_f "collapse_until %s" tfs >>= fun () ->
  let cn_store = Filename.concat tlog_dir head_name in
  Local_store.make_local_store cn_store >>= fun store ->
  store # consensus_i () >>= fun store_i ->
  let start_i = 
    match store_i with
      | None -> Sn.start
      | Some i -> Sn.succ i
  in
  if start_i >= too_far_i 
  then
    let msg = Printf.sprintf "Store counter (%s) is ahead of end point (%s). Aborting"
      (Sn.string_of start_i) tfs in
    Lwt.fail (Failure msg) 
  else
    begin
      let acc = ref None in
      let add_to_store (i,update) = 
	match !acc with
	  | None ->
	    let () = acc := Some(i,update) in
	    Lwt_log.debug_f "update %s has no previous" (Sn.string_of i) >>= fun () ->
	    Lwt.return ()
	      | Some (pi,pu) ->
		if pi < i then
		  begin
		    Lwt_log.debug_f "%s => store" (Sn.string_of pi) >>= fun () ->
		    Store.safe_insert_update store pi pu >>= fun _ ->
		    let () = acc := Some(i,update) in
		    Lwt.return ()
		  end
		else
		  begin
		    Lwt_log.debug_f "%s => skip" (Sn.string_of pi) >>= fun () ->
		    let () = acc := Some(i,update) in
		    Lwt.return ()
		  end
      in
      Lwt_log.debug_f "collapse_until: start_i=%s" (Sn.string_of start_i) >>= fun () ->
      Tlc2.iterate_tlog_dir tlog_dir start_i too_far_i add_to_store >>= fun () ->
      store # close ()
    end



let mv_file source target = 
  Unix.rename source target; 
  Lwt.return ()

let unlink_file target = 
  Unix.unlink target; 
  Lwt.return ()

let collapse_many tlog_dir tlog_names head_name = 
  Lwt_log.debug_f "collapse_many" >>= fun () ->
  let new_name = head_name ^ ".next" in
  let cn1 = Filename.concat tlog_dir head_name 
  and cn2 = Filename.concat tlog_dir new_name 
  in
  Lwt.catch
    (fun () -> File_system.copy_file cn1 cn2)<commit_msg>cleanup indentation<commit_after>(*
This file is part of Arakoon, a distributed key-value store. Copyright
(C) 2010 Incubaid BVBA

Licensees holding a valid Incubaid license may use this file in
accordance with Incubaid's Arakoon commercial license agreement. For
more information on how to enter into this agreement, please contact
Incubaid (contact details can be found on www.arakoon.org/licensing).

Alternatively, this file may be redistributed and/or modified under
the terms of the GNU Affero General Public License version 3, as
published by the Free Software Foundation. Under this license, this
file is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.

See the GNU Affero General Public License for more details.
You should have received a copy of the
GNU Affero General Public License along with this program (file "COPYING").
If not, see <http://www.gnu.org/licenses/>.
*)

open Lwt

let head_name = Tlogcollection.head_name

let collapse_until tlog_dir head_name too_far_i =
  let tfs = Sn.string_of too_far_i in
  Lwt_log.debug_f "collapse_until %s" tfs >>= fun () ->
  let cn_store = Filename.concat tlog_dir head_name in
  Local_store.make_local_store cn_store >>= fun store ->
  store # consensus_i () >>= fun store_i ->
  let start_i = 
    match store_i with
      | None -> Sn.start
      | Some i -> Sn.succ i
  in
  if start_i >= too_far_i 
  then
    let msg = Printf.sprintf 
      "Store counter (%s) is ahead of end point (%s). Aborting"
      (Sn.string_of start_i) tfs in
    Lwt.fail (Failure msg) 
  else
    begin
      let acc = ref None in
      let add_to_store (i,update) = 
	match !acc with
	  | None ->
	    begin
	      let () = acc := Some(i,update) in
	      Lwt_log.debug_f "update %s has no previous" (Sn.string_of i) 
	      >>= fun () ->
	      Lwt.return ()
	    end
	  | Some (pi,pu) ->
	    if pi < i then
	      begin
		Lwt_log.debug_f "%s => store" (Sn.string_of pi) >>= fun () ->
		Store.safe_insert_update store pi pu >>= fun _ ->
		let () = acc := Some(i,update) in
		Lwt.return ()
	      end
	    else
	      begin
		Lwt_log.debug_f "%s => skip" (Sn.string_of pi) >>= fun () ->
		let () = acc := Some(i,update) in
		Lwt.return ()
	      end
      in
      Lwt_log.debug_f "collapse_until: start_i=%s" (Sn.string_of start_i) 
      >>= fun () ->
      Tlc2.iterate_tlog_dir tlog_dir start_i too_far_i add_to_store 
      >>= fun () ->
      store # close ()
    end
      


let mv_file source target = 
  Unix.rename source target; 
  Lwt.return ()

let unlink_file target = 
  Unix.unlink target; 
  Lwt.return ()

let collapse_many tlog_dir tlog_names head_name = 
  Lwt_log.debug_f "collapse_many" >>= fun () ->
  let new_name = head_name ^ ".next" in
  let cn1 = Filename.concat tlog_dir head_name 
  and cn2 = Filename.concat tlog_dir new_name 
  in
  Lwt.catch
    (fun () -> File_system.copy_file cn1 cn2)
{'fraction_non_alphanumeric': 0.09277128830201456, 'fraction_numerical': 0.0033858134416793633, 'mean_word_length': 3.032764505119454, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1511384', 'n_tokens_mistral': 2062, 'n_tokens_neox': 1949, 'n_words': 684}