<commit_before>let (+) = Filename.concat
open Types

let log fmt = Globals.log "RSYNC" fmt

let rsync ?(delete=true) src dst =
  log "rsync: delete:%b src:%s dst:%s" delete src dst;
  if src <> dst then (
    Run.mkdir src;
    Run.mkdir dst;
    let delete = if delete then ["--delete"] else [] in
    match
      Run.read_command_output (["rsync" ; "-arv"; src; dst] @ delete)
    with
    | None   -> Not_available
    | Some l -> match Utils.rsync_trim l with
      | []    -> Up_to_date []
      | lines ->
          List.iter (fun f -> log "updated: %s %s" (Run.cwd ()) f) lines;
          Result lines
  ) else
    Up_to_date []

let rsync_dirs ?delete src dst =
  let src_s = Dirname.to_string src + "" in
  let dst_s = Dirname.to_string dst in
  let dst_files0 = Filename.rec_list dst in
  match rsync ?delete src_s dst_s with
  | Not_available -> Not_available
  | Up_to_date _  -> Up_to_date []
  | Result lines  ->
      let src_files = Filename.rec_list src in
      let dst_files = Filename.rec_list dst in
      if delete = Some true && List.length src_files <> List.length dst_files then (
        List.iter (fun f -> Globals.msg "src-file: %s\n" (Filename.to_string f)) src_files;
        List.iter (fun f -> Globals.msg "dst-file0: %s\n" (Filename.to_string f)) dst_files0;
        List.iter (fun f -> Globals.msg "dst-file: %s\n" (Filename.to_string f)) dst_files;
        Globals.error_and_exit "rsync_dir failed!"
      );
      Result lines

let rsync_file src dst =
  match
    Run.read_command_output [
      "rsync"; "-av"; Filename.to_string src; Filename.to_string dst;
    ]
  with
  | None   -> Not_available
  | Some l -> match Utils.rsync_trim l with
    | []  -> Up_to_date dst
    | [x] -> Result dst
    | l   ->
        Globals.error_and_exit
          "unknown rsync output: {%s}"
          (String.concat ", " l)

module B = struct

  let init r = ()<commit_msg>[rsync] add more debug messages<commit_after>let (+) = Filename.concat
open Types

let log fmt = Globals.log "RSYNC" fmt

let rsync ?(delete=true) src dst =
  log "rsync: delete:%b src:%s dst:%s" delete src dst;
  if src <> dst then (
    Run.mkdir src;
    Run.mkdir dst;
    let delete = if delete then ["--delete"] else [] in
    match
      Run.read_command_output (["rsync" ; "-arv"; src; dst] @ delete)
    with
    | None   -> Not_available
    | Some l -> match Utils.rsync_trim l with
      | []    -> Up_to_date []
      | lines ->
          List.iter (fun f -> log "updated: %s %s" (Run.cwd ()) f) lines;
          Result lines
  ) else
    Up_to_date []

let rsync_dirs ?delete src dst =
  let src_s = Dirname.to_string src + "" in
  let dst_s = Dirname.to_string dst in
  let dst_files0 = Filename.rec_list dst in
  match rsync ?delete src_s dst_s with
  | Not_available -> Not_available
  | Up_to_date _  -> Up_to_date []
  | Result lines  ->
      let src_files = Filename.rec_list src in
      let dst_files = Filename.rec_list dst in
      if delete = Some true && List.length src_files <> List.length dst_files then (
        List.iter (fun f -> Globals.msg "src-file: %s\n" (Filename.to_string f)) src_files;
        List.iter (fun f -> Globals.msg "dst-file0: %s\n" (Filename.to_string f)) dst_files0;
        List.iter (fun f -> Globals.msg "dst-file: %s\n" (Filename.to_string f)) dst_files;
        Globals.error_and_exit "rsync_dir failed!"
      );
      Result lines

let rsync_file src dst =
  log "rsync_file src=%s dst=%s" (Filename.to_string src) (Filename.to_string dst);
  match
    Run.read_command_output [
      "rsync"; "-av"; Filename.to_string src; Filename.to_string dst;
    ]
  with
  | None   -> Not_available
  | Some l -> match Utils.rsync_trim l with
    | []  -> Up_to_date dst
    | [x] -> Result dst
    | l   ->
        Globals.error_and_exit
          "unknown rsync output: {%s}"
          (String.concat ", " l)

module B = struct

  let init r = ()
{'fraction_non_alphanumeric': 0.12384615384615384, 'fraction_numerical': 0.0015384615384615385, 'mean_word_length': 2.486148346738159, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '720039', 'n_tokens_mistral': 1399, 'n_tokens_neox': 1385, 'n_words': 400}