<commit_before>(*
Copyright 2015 iNuron NV

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*)

open Prelude
open Nsm_model
open Lwt.Infix
open OUnit
open Cmdliner

let cfg_url = ref (Url.File "./cfg/test.ini")
let tls_config = ref None

let _easiest_upload () =
  let tls_config = !tls_config
  and namespace = "demo"
  and namespace_id = 0l
  and input_file = "./ocaml/alba.native"
  and object_name = Printf.sprintf "easy_test_%i" 1
  and allow_overwrite = false in
  let shoot (sid,sinfo) =
    let open Nsm_model in
    match sinfo.OsdInfo.kind with
      | OsdInfo.Asd (conn_info,long_id) ->
         begin
           let ips,port,_,_ = conn_info in
           let ip = List.hd_exn ips in
           Lwt_io.printlf "going to kill ASD: %li (%s,%i)"  sid ip port
           >>=fun()->
           let cmd = Printf.sprintf "pkill -f 'alba.native.*%02li.*'" sid in
           Printf.printf "cmd=%S\n%!" cmd;
           let rc = Sys.command cmd in
           Lwt_io.printlf "rc=%i" rc
         end
      | OsdInfo.Kinetic(conn_info, long_id) ->
         begin
           let ips,port,_,_ = conn_info in
           let ip = List.hd_exn ips in
           Lwt_io.printlf "going to kill Kinetic: %li (%s,%i)" sid ip port
           >>= fun () ->
           let cmd = Printf.sprintf "pkill -f 'java.*-port %i.*'" port in
           let rc = Sys.command cmd in
           Lwt_io.printlf "rc=%i" rc
         end
  in
  Alba_arakoon.config_from_url !cfg_url >>= fun cfg ->
  Alba_client.with_client
    (ref cfg)
    ~tls_config
    (fun alba_client ->

     Alba_test._wait_for_osds ~cnt:6 alba_client namespace_id >>= fun () ->
     alba_client # mgr_access # list_all_claimed_osds >>= fun (n, osds) ->
     Lwt_io.printlf "there are n=%i claimed osds" n >>= fun()->
     let soon_dead = List.hd_exn osds
     in
     shoot soon_dead >>=fun()->
     Lwt_io.printlf "uploading"
     >>=fun()->

     alba_client # upload_object_from_file
       ~namespace
       ~object_name ~input_file
       ~checksum_o:None
       ~allow_overwrite:(if allow_overwrite
                         then Unconditionally
                         else NoPrevious)
     >>= fun (manifest, _) ->
     Lwt_io.printlf "manifest=%s" ([%show : Nsm_model.Manifest.t] manifest)
     >>= fun()->
     let open Manifest in
     let chunks = manifest.fragment_locations in
     let osd_id_o, version_id = List.hd_exn (List.hd_exn chunks) in
     Lwt_io.printlf "osd_id:%li" (Option.get_some osd_id_o))


let easiest_upload ctx =
  Lwt_engine.set (new Lwt_rsocket.rselect);
  let t =
    Lwt.finalize
      (fun () ->
        Lwt_log.file 
          ~file_name:"/tmp/disk_failure_tests.log"
          ~template:"$(date).$(milliseconds) $(message)"
          () >>= fun logger ->
        Lwt_log.default := logger;
        Lwt_log_core.append_rule "*" Lwt_log_core.Debug;
        _easiest_upload())
      (fun () ->
        Alba_wrappers.Sys2.lwt_get_maxrss () >>= fun maxrss ->
        let stat = Gc.quick_stat () in
        let factor = (float (Sys.word_size / 8)) /. 1024.0 in
        let mem_allocated = (stat.Gc.minor_words +. stat.Gc.major_words -. stat.Gc.promoted_words)
                            *. factor in
        let msg = Printf.sprintf
                    "maxrss:%i KB, allocated:%f, minor_collections:%i, major_collections:%i, compactions:%i, heap_words:%i"
                    maxrss mem_allocated stat.Gc.minor_collections
                    stat.Gc.major_collections stat.Gc.compactions
                    stat.Gc.heap_words
        in
        Lwt_log.info msg >>= fun () ->
        Lwt_io.printlf "%s%!" msg 
        
      )
  in
  Lwt_main.run t

let () =
  let suite =
    "disk_failures" >:::
       ["easiest_upload" >:: easiest_upload;]

  in
  let run_unit_tests alba_cfg_url tls_config' produce_xml =
    let () = cfg_url := alba_cfg_url in
    let () = tls_config := tls_config' in
    let _results =
      if produce_xml
      then OUnit_XML.run_suite_producing_xml suite "testresults.xml"
      else OUnit.run_test_tt_main suite
    in
    ()
  in<commit_msg>travis: before and after...<commit_after>(*
Copyright 2015 iNuron NV

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*)

open Prelude
open Nsm_model
open Lwt.Infix
open OUnit
open Cmdliner

let cfg_url = ref (Url.File "./cfg/test.ini")
let tls_config = ref None

let _easiest_upload () =
  let tls_config = !tls_config
  and namespace = "demo"
  and namespace_id = 0l
  and input_file = "./ocaml/alba.native"
  and object_name = Printf.sprintf "easy_test_%i" 1
  and allow_overwrite = false in
  let shoot (sid,sinfo) =
    let open Nsm_model in
    match sinfo.OsdInfo.kind with
      | OsdInfo.Asd (conn_info,long_id) ->
         begin
           let ips,port,_,_ = conn_info in
           let ip = List.hd_exn ips in
           Lwt_io.printlf "going to kill ASD: %li (%s,%i)"  sid ip port
           >>=fun()->
           let cmd = Printf.sprintf "pkill -f 'alba.native.*%02li.*'" sid in
           Printf.printf "cmd=%S\n%!" cmd;
           let rc = Sys.command cmd in
           Lwt_io.printlf "rc=%i" rc
         end
      | OsdInfo.Kinetic(conn_info, long_id) ->
         begin
           let ips,port,_,_ = conn_info in
           let ip = List.hd_exn ips in
           Lwt_io.printlf "going to kill Kinetic: %li (%s,%i)" sid ip port
           >>= fun () ->
           let cmd = Printf.sprintf "pkill -f 'java.*-port %i.*'" port in
           let rc = Sys.command cmd in
           Lwt_io.printlf "rc=%i" rc
         end
  in
  Alba_arakoon.config_from_url !cfg_url >>= fun cfg ->
  Alba_client.with_client
    (ref cfg)
    ~tls_config
    (fun alba_client ->

     Alba_test._wait_for_osds ~cnt:6 alba_client namespace_id >>= fun () ->
     alba_client # mgr_access # list_all_claimed_osds >>= fun (n, osds) ->
     Lwt_io.printlf "there are n=%i claimed osds" n >>= fun()->
     let soon_dead = List.hd_exn osds
     in
     shoot soon_dead >>=fun()->
     Lwt_io.printlf "uploading"
     >>=fun()->

     alba_client # upload_object_from_file
       ~namespace
       ~object_name ~input_file
       ~checksum_o:None
       ~allow_overwrite:(if allow_overwrite
                         then Unconditionally
                         else NoPrevious)
     >>= fun (manifest, _) ->
     Lwt_io.printlf "manifest=%s" ([%show : Nsm_model.Manifest.t] manifest)
     >>= fun()->
     let open Manifest in
     let chunks = manifest.fragment_locations in
     let osd_id_o, version_id = List.hd_exn (List.hd_exn chunks) in
     Lwt_io.printlf "osd_id:%li" (Option.get_some osd_id_o))


let easiest_upload ctx =
  Lwt_engine.set (new Lwt_rsocket.rselect);
  let dump_stats () =
    Alba_wrappers.Sys2.lwt_get_maxrss () >>= fun maxrss ->
        let stat = Gc.quick_stat () in
        let factor = (float (Sys.word_size / 8)) /. 1024.0 in
        let mem_allocated = (stat.Gc.minor_words +. stat.Gc.major_words -. stat.Gc.promoted_words)
                            *. factor in
        let msg = Printf.sprintf
                    "maxrss:%i KB, allocated:%f, minor_collections:%i, major_collections:%i, compactions:%i, heap_words:%i"
                    maxrss mem_allocated stat.Gc.minor_collections
                    stat.Gc.major_collections stat.Gc.compactions
                    stat.Gc.heap_words
        in
        Lwt_log.info msg >>= fun () ->
        Lwt_io.printlf "%s%!" msg
  in
  let t =
    Lwt.finalize
      (fun () ->
        dump_stats () >>= fun () ->
        Lwt_log.file 
          ~file_name:"/tmp/disk_failure_tests.log"
          ~template:"$(date).$(milliseconds) $(message)"
          () >>= fun logger ->
        Lwt_log.default := logger;
        Lwt_log_core.append_rule "*" Lwt_log_core.Debug;
        _easiest_upload())
      (fun () ->
        dump_stats ()
        
      )
  in
  Lwt_main.run t

let () =
  let suite =
    "disk_failures" >:::
       ["easiest_upload" >:: easiest_upload;]

  in
  let run_unit_tests alba_cfg_url tls_config' produce_xml =
    let () = cfg_url := alba_cfg_url in
    let () = tls_config := tls_config' in
    let _results =
      if produce_xml
      then OUnit_XML.run_suite_producing_xml suite "testresults.xml"
      else OUnit.run_test_tt_main suite
    in
    ()
  in
{'fraction_non_alphanumeric': 0.10722865254422363, 'fraction_numerical': 0.0043677658877484165, 'mean_word_length': 2.292235801581596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 98, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3494374', 'n_tokens_mistral': 3182, 'n_tokens_neox': 3065, 'n_words': 810}