<commit_before>(* Handy tool for managing CSV files.
   @author <NAME> <<EMAIL>>
*)

open Printf
open Csv

(*------------------------------ start of code from extlib *)
exception Invalid_string

let find str sub =
  let sublen = String.length sub in
  if sublen = 0 then
    0
  else
    let found = ref 0 in
    let len = String.length str in
    try
      for i = 0 to len - sublen do
        let j = ref 0 in
        while String.unsafe_get str (i + !j) = String.unsafe_get sub !j do
          incr j;
          if !j = sublen then begin found := i; raise Exit; end;
        done;
      done;
      raise Invalid_string
    with
      Exit -> !found

let split str sep =
  let p = find str sep in
  let len = String.length sep in
  let slen = String.length str in
  String.sub str 0 p, String.sub str (p + len) (slen - p - len)

let nsplit str sep =
  if str = "" then []
  else (
    let rec nsplit str sep =
      try
        let s1 , s2 = split str sep in
        s1 :: nsplit s2 sep
      with
        Invalid_string -> [str]
    in
    nsplit str sep
  )

type 'a mut_list =  {
        hd: 'a;
        mutable tl: 'a list
}
external inj : 'a mut_list -> 'a list = "%identity"

let dummy_node () = { hd = Obj.magic (); tl = [] }

let rec drop n = function
  | _ :: l when n > 0 -> drop (n-1) l
  | l -> l

let take n l =
  let rec loop n dst = function
    | h :: t when n > 0 ->
        let r = { hd = h; tl = [] } in
        dst.tl <- inj r;
        loop (n-1) r t
    | _ ->
        ()
  in
  let dummy = dummy_node() in
  loop n dummy l;
  dummy.tl
(*------------------------------ end of extlib code *)

(* Parse column specs. *)
type colspec = range list
and range =
  | Col of int (* 0 *)
  | Range of int * int (* 2-5 *)
  | ToEnd of int (* 7- *)

let parse_colspec ~count_zero colspec =
  let cols = nsplit colspec "," in
  let cols = List.map (
    fun col ->
      try
        (try
           let first, second = split col "-" in
           if second <> "" then
             Range (int_of_string first, int_of_string second)
           else
             ToEnd (int_of_string first)
         with
           Invalid_string ->
             Col (int_of_string col)
        )
      with
        Failure "int_of_string" ->
          failwith (colspec ^ ":" ^ col ^ ": invalid column-spec")
  ) cols in

  (* Adjust so columns always count from zero. *)
  if not count_zero then
    List.map (
      function
      | Col c -> Col (c-1)
      | Range (s, e) -> Range (s-1, e-1)
      | ToEnd e -> ToEnd (e-1)
    ) cols
  else
    cols

let rec width_of_colspec = function
  | [] -> 0
  | Col c :: rest -> 1 + width_of_colspec rest
  | Range (s, e) :: rest -> (e-s+1) + width_of_colspec rest
  | ToEnd _ :: _ ->
      failwith "width_of_colspec: cannot calculate width of an open column spec (one which contains 'N-')"

(* For closed column specs, this preserves the correct width in the
 * result.
 *)
let cols_of_colspec colspec row =
  let rec loop = function
    | [] -> []
    | Col c :: rest ->
        (try List.nth row c
         with Failure "nth" -> "") :: loop rest
    | Range (s, e) :: rest ->
        let width = e-s+1 in
        let range = take width (drop s row) in
        let range = List.hd (set_columns width [range]) in
        List.append range (loop rest)
    | ToEnd e :: rest ->
        List.append (drop e row) (loop rest)
  in
  loop colspec

(* The actual commands. *)
let cmd_cols ~input_sep ~output_sep ~chan colspec files =
  List.iter (
    fun filename ->
      let csv = load ~separator:input_sep filename in
      let csv = List.map (cols_of_colspec colspec) csv in
      save_out ~separator:output_sep chan csv
  ) files

let cmd_namedcols ~input_sep ~output_sep ~chan names files =
  List.iter (
    fun filename ->
      let csv = load ~separator:input_sep filename in
      let header, data =
        match csv with
        | [] -> failwith "no rows in this CSV file"
        | h :: t -> h, t in
      (* Do the headers requested exist in the CSV file?  If not,
       * throw an error.
       *)
      List.iter (
        fun name ->
          if not (List.mem name header) then
            failwith ("namedcol: requested header not in CSV file: " ^ name)
      ) names;
      let data = associate header data in
      let data = List.map (
        fun row -> List.map (fun name -> List.assoc name row) names
      ) data in
      save_out ~separator:output_sep chan data
  ) files

let cmd_width ~input_sep ~chan files =
  let width = List.fold_left (
    fun width filename ->
      let csv = load ~separator:input_sep filename in
      let width = max width (columns csv) in
      width
  ) 0 files in
  fprintf chan "%d\n" width

let cmd_height ~input_sep ~chan files =
  let height = List.fold_left (
    fun height filename ->
      let csv = load ~separator:input_sep filename in
      let height = height + lines csv in
      height
  ) 0 files in
  fprintf chan "%d\n" height

let cmd_readable ~input_sep ~chan files =
  let csv = List.concat (List.map (load ~separator:input_sep) files) in
  save_out_readable chan csv

let cmd_cat ~input_sep ~output_sep ~chan files =
  (* Avoid loading the whole file into memory. *)
  let f row =
    save_out ~separator:output_sep chan [row]
  in
  List.iter (
    fun filename ->
      let in_chan, close =
        match filename with
        | "-" -> stdin, false
        | filename -> open_in filename, true in
      load_rows ~separator:input_sep f in_chan;
      if close then close_in in_chan
  ) files

let cmd_set_columns ~input_sep ~output_sep ~chan cols files =
  (* Avoid loading the whole file into memory. *)
  let f row =
    let csv = [row] in
    let csv = set_columns cols csv in
    save_out ~separator:output_sep chan csv
  in
  List.iter (
    fun filename ->
      let in_chan, close =
        match filename with
        | "-" -> stdin, false
        | filename -> open_in filename, true in
      load_rows ~separator:input_sep f in_chan;
      if close then close_in in_chan
  ) files

let cmd_set_rows ~input_sep ~output_sep ~chan rows files =
  let csv = List.concat (List.map (load ~separator:input_sep) files) in
  let csv = set_rows rows csv in
  save_out ~separator:output_sep chan csv

let cmd_head ~input_sep ~output_sep ~chan rows files =
  (* Avoid loading the whole file into memory, or even loading
   * later files.
   *)
  let nr_rows = ref rows in
  let f row =
    if !nr_rows > 0 then (
      decr nr_rows;
      save_out ~separator:output_sep chan [row]
    )
  in
  List.iter (
    fun filename ->
      if !nr_rows > 0 then (
        let in_chan, close =
          match filename with
          | "-" -> stdin, false
          | filename -> open_in filename, true in
        load_rows ~separator:input_sep f in_chan;
        if close then close_in in_chan
      )
  ) files

let cmd_drop ~input_sep ~output_sep ~chan rows files =
  (* Avoid loading the whole file into memory. *)
  let nr_rows = ref rows in
  let f row =
    if !nr_rows = 0 then
      save_out ~separator:output_sep chan [row]
    else
      decr nr_rows
  in
  List.iter (
    fun filename ->
      let in_chan, close =
        match filename with
        | "-" -> stdin, false
        | filename -> open_in filename, true in
      load_rows ~separator:input_sep f in_chan;
      if close then close_in in_chan
  ) files

let cmd_square ~input_sep ~output_sep ~chan files =
  let csv = List.concat (List.map (load ~separator:input_sep) files) in
  let csv = square csv in
  save_out ~separator:output_sep chan csv

let cmd_sub ~input_sep ~output_sep ~chan r c rows cols files =
  let csv = List.concat (List.map (load ~separator:input_sep) files) in
  let csv = sub r c rows cols csv in
  save_out ~separator:output_sep chan csv

let cmd_replace ~input_sep ~output_sep ~chan colspec update files =
  let csv = List.concat (List.map (load ~separator:input_sep) files) in

  (* Load the update CSV file in. *)
  let update = Csv.load ~separator:input_sep update in

  (* Compare two rows for equality by considering only the columns
   * in colspec.
   *)
  let equal row1 row2 =
    let row1 = cols_of_colspec colspec row1 in
    let row2 = cols_of_colspec colspec row2 in
    0 = Csv.compare [row1] [row2]
  in

  (* Look for rows in the original to be replaced by rows from the
   * update file.  This is an ugly O(n^2) hack (XXX).
   *)
  let csv = List.filter (
    fun row -> not (List.exists (equal row) update)
  ) csv in
  let csv = csv @ update in
  save_out ~separator:output_sep chan csv

let cmd_call ~input_sep ~output_sep ~chan command files =
  (* Avoid loading the whole file into memory. *)
  let f row =
    let cmd =
      command ^ " " ^ String.concat " " (List.map Filename.quote row) in
    let code = Sys.command cmd in
    if code <> 0 then (
      eprintf "%s: terminated with exit code %d\n" command code;
      exit code
    )
  in
  List.iter (
    fun filename ->
      let in_chan, close =
        match filename with
        | "-" -> stdin, false
        | filename -> open_in filename, true in
      load_rows ~separator:input_sep f in_chan;
      if close then close_in in_chan
  ) files

let rec uniq = function
  | [] -> []
  | [x] -> [x]
  | x :: y :: xs when Pervasives.compare x y = 0 ->
      uniq (x :: xs)
  | x :: y :: xs ->
      x :: uniq (y :: xs)

let cmd_join ~input_sep ~output_sep ~chan colspec1 colspec2 files =
  (* Load in the files separately. *)
  let csvs = List.map (load ~separator:input_sep) files in

  (* For each CSV file, construct a hash table from row class (key) to
   * the (possibly empty) output columns (values).
   * Also construct a hash which has the unique list of row classes.
   *)
  let keys = Hashtbl.create 1023 in
  let hashes = List.map (
    fun csv ->
      let hash = Hashtbl.create 1023 in
      List.iter (
        fun row ->
          let key = cols_of_colspec colspec1 row in
          let value = cols_of_colspec colspec2 row in
          if not (Hashtbl.mem keys key) then Hashtbl.add keys key true;
          Hashtbl.add hash key value
      ) csv;
      hash
  ) csvs in

  (* Get the keys. *)
  let keys = Hashtbl.fold (fun key _ xs -> key :: xs) keys [] in

  let value_width = width_of_colspec colspec2 in
  let empty_value =
    List.hd (set_columns value_width [[""]]) in
  let multiple_values =
    List.hd (set_columns value_width [["!MULTIPLE VALUES"]]) in

  (* Generate output CSV. *)
  let keys = List.sort Pervasives.compare keys in
  let keys = List.map (fun key -> key, []) keys in
  let csv = List.fold_left (
    fun keys hash ->
      List.map (
        fun (key, values) ->
          let value = try Hashtbl.find_all hash key with Not_found -> [] in
          let value =
            match value with
            | [] -> empty_value
            | [value] -> value
            | _::_ -> multiple_values in
          key, (value :: values)
      ) keys
  ) keys hashes in
  let csv = List.map (
    fun (key, values) ->
      key @ List.flatten (List.rev values)
  ) csv in
  save_out ~separator:output_sep chan csv

let rec cmd_trim ~input_sep ~output_sep ~chan (top, left, right, bottom) files =
  let csv = List.concat (List.map (load ~separator:input_sep) files) in
  let csv = trim ~top ~left ~right ~bottom csv in
  save_out ~separator:output_sep chan csv<commit_msg>Modified the cvstool so it does not use deprecated features (and qualifies all uses of the Csv functions).<commit_after>(* Handy tool for managing CSV files.
   @author <NAME> <<EMAIL>>
*)

open Printf

(*------------------------------ start of code from extlib *)
exception Invalid_string

let find str sub =
  let sublen = String.length sub in
  if sublen = 0 then
    0
  else
    let found = ref 0 in
    let len = String.length str in
    try
      for i = 0 to len - sublen do
        let j = ref 0 in
        while String.unsafe_get str (i + !j) = String.unsafe_get sub !j do
          incr j;
          if !j = sublen then begin found := i; raise Exit; end;
        done;
      done;
      raise Invalid_string
    with
      Exit -> !found

let split str sep =
  let p = find str sep in
  let len = String.length sep in
  let slen = String.length str in
  String.sub str 0 p, String.sub str (p + len) (slen - p - len)

let nsplit str sep =
  if str = "" then []
  else (
    let rec nsplit str sep =
      try
        let s1 , s2 = split str sep in
        s1 :: nsplit s2 sep
      with
        Invalid_string -> [str]
    in
    nsplit str sep
  )

type 'a mut_list =  {
        hd: 'a;
        mutable tl: 'a list
}
external inj : 'a mut_list -> 'a list = "%identity"

let dummy_node () = { hd = Obj.magic (); tl = [] }

let rec drop n = function
  | _ :: l when n > 0 -> drop (n-1) l
  | l -> l

let take n l =
  let rec loop n dst = function
    | h :: t when n > 0 ->
        let r = { hd = h; tl = [] } in
        dst.tl <- inj r;
        loop (n-1) r t
    | _ ->
        ()
  in
  let dummy = dummy_node() in
  loop n dummy l;
  dummy.tl
(*------------------------------ end of extlib code *)

(* Parse column specs. *)
type colspec = range list
and range =
  | Col of int (* 0 *)
  | Range of int * int (* 2-5 *)
  | ToEnd of int (* 7- *)

let parse_colspec ~count_zero colspec =
  let cols = nsplit colspec "," in
  let cols = List.map (
    fun col ->
      try
        (try
           let first, second = split col "-" in
           if second <> "" then
             Range (int_of_string first, int_of_string second)
           else
             ToEnd (int_of_string first)
         with
           Invalid_string ->
             Col (int_of_string col)
        )
      with
        Failure "int_of_string" ->
          failwith (colspec ^ ":" ^ col ^ ": invalid column-spec")
  ) cols in

  (* Adjust so columns always count from zero. *)
  if not count_zero then
    List.map (
      function
      | Col c -> Col (c-1)
      | Range (s, e) -> Range (s-1, e-1)
      | ToEnd e -> ToEnd (e-1)
    ) cols
  else
    cols

let rec width_of_colspec = function
  | [] -> 0
  | Col c :: rest -> 1 + width_of_colspec rest
  | Range (s, e) :: rest -> (e-s+1) + width_of_colspec rest
  | ToEnd _ :: _ ->
      failwith "width_of_colspec: cannot calculate width of an open column spec (one which contains 'N-')"

(* For closed column specs, this preserves the correct width in the
 * result.
 *)
let cols_of_colspec colspec row =
  let rec loop = function
    | [] -> []
    | Col c :: rest ->
        (try List.nth row c
         with Failure "nth" -> "") :: loop rest
    | Range (s, e) :: rest ->
        let width = e-s+1 in
        let range = take width (drop s row) in
        let range = List.hd (Csv.set_columns width [range]) in
        List.append range (loop rest)
    | ToEnd e :: rest ->
        List.append (drop e row) (loop rest)
  in
  loop colspec

(* The actual commands. *)
let cmd_cols ~input_sep ~output_sep ~chan colspec files =
  List.iter (
    fun filename ->
      let csv = Csv.load ~separator:input_sep filename in
      let csv = List.map (cols_of_colspec colspec) csv in
      Csv.output_all (Csv.to_channel ~separator:output_sep chan) csv
  ) files

let cmd_namedcols ~input_sep ~output_sep ~chan names files =
  List.iter (
    fun filename ->
      let csv = Csv.load ~separator:input_sep filename in
      let header, data =
        match csv with
        | [] -> failwith "no rows in this CSV file"
        | h :: t -> h, t in
      (* Do the headers requested exist in the CSV file?  If not,
       * throw an error.
       *)
      List.iter (
        fun name ->
          if not (List.mem name header) then
            failwith ("namedcol: requested header not in CSV file: " ^ name)
      ) names;
      let data = Csv.associate header data in
      let data = List.map (
        fun row -> List.map (fun name -> List.assoc name row) names
      ) data in
      Csv.output_all (Csv.to_channel ~separator:output_sep chan) data
  ) files

let cmd_width ~input_sep ~chan files =
  let width = List.fold_left (
    fun width filename ->
      let csv = Csv.load ~separator:input_sep filename in
      let width = max width (Csv.columns csv) in
      width
  ) 0 files in
  fprintf chan "%d\n" width

let cmd_height ~input_sep ~chan files =
  let height = List.fold_left (
    fun height filename ->
      let csv = Csv.load ~separator:input_sep filename in
      let height = height + Csv.lines csv in
      height
  ) 0 files in
  fprintf chan "%d\n" height

let cmd_readable ~input_sep ~chan files =
  let csv = List.concat (List.map (Csv.load ~separator:input_sep) files) in
  Csv.save_out_readable chan csv

let cmd_cat ~input_sep ~output_sep ~chan files =
  (* Avoid loading the whole file into memory. *)
  let chan = Csv.to_channel ~separator:output_sep chan in
  let f row =
    Csv.output_record chan row
  in
  List.iter (
    fun filename ->
      let in_chan, close =
        match filename with
        | "-" -> stdin, false
        | filename -> open_in filename, true in
      Csv.iter f (Csv.of_channel ~separator:input_sep in_chan);
      if close then close_in in_chan
  ) files

let cmd_set_columns ~input_sep ~output_sep ~chan cols files =
  (* Avoid loading the whole file into memory. *)
  let f row =
    let csv = [row] in
    let csv = Csv.set_columns cols csv in
    Csv.output_all (Csv.to_channel ~separator:output_sep chan) csv
  in
  List.iter (
    fun filename ->
      let in_chan, close =
        match filename with
        | "-" -> stdin, false
        | filename -> open_in filename, true in
      Csv.iter f (Csv.of_channel ~separator:input_sep in_chan);
      if close then close_in in_chan
  ) files

let cmd_set_rows ~input_sep ~output_sep ~chan rows files =
  let csv = List.concat (List.map (Csv.load ~separator:input_sep) files) in
  let csv = Csv.set_rows rows csv in
  Csv.output_all (Csv.to_channel ~separator:output_sep chan) csv

let cmd_head ~input_sep ~output_sep ~chan rows files =
  (* Avoid loading the whole file into memory, or even loading
   * later files.
   *)
  let nr_rows = ref rows in
  let chan = Csv.to_channel ~separator:output_sep chan in
  let f row =
    if !nr_rows > 0 then (
      decr nr_rows;
      Csv.output_record chan row
    )
  in
  List.iter (
    fun filename ->
      if !nr_rows > 0 then (
        let in_chan, close =
          match filename with
          | "-" -> stdin, false
          | filename -> open_in filename, true in
        Csv.iter f (Csv.of_channel ~separator:input_sep in_chan);
        if close then close_in in_chan
      )
  ) files

let cmd_drop ~input_sep ~output_sep ~chan rows files =
  (* Avoid loading the whole file into memory. *)
  let nr_rows = ref rows in
  let chan = Csv.to_channel ~separator:output_sep chan in
  let f row =
    if !nr_rows = 0 then
      Csv.output_record chan row
    else
      decr nr_rows
  in
  List.iter (
    fun filename ->
      let in_chan, close =
        match filename with
        | "-" -> stdin, false
        | filename -> open_in filename, true in
      Csv.iter f (Csv.of_channel ~separator:input_sep in_chan);
      if close then close_in in_chan
  ) files

let cmd_square ~input_sep ~output_sep ~chan files =
  let csv = List.concat (List.map (Csv.load ~separator:input_sep) files) in
  let csv = Csv.square csv in
  Csv.output_all (Csv.to_channel ~separator:output_sep chan) csv

let cmd_sub ~input_sep ~output_sep ~chan r c rows cols files =
  let csv = List.concat (List.map (Csv.load ~separator:input_sep) files) in
  let csv = Csv.sub r c rows cols csv in
  Csv.output_all (Csv.to_channel ~separator:output_sep chan) csv

let cmd_replace ~input_sep ~output_sep ~chan colspec update files =
  let csv = List.concat (List.map (Csv.load ~separator:input_sep) files) in

  (* Load the update CSV file in. *)
  let update = Csv.load ~separator:input_sep update in

  (* Compare two rows for equality by considering only the columns
   * in colspec.
   *)
  let equal row1 row2 =
    let row1 = cols_of_colspec colspec row1 in
    let row2 = cols_of_colspec colspec row2 in
    0 = Csv.compare [row1] [row2]
  in

  (* Look for rows in the original to be replaced by rows from the
   * update file.  This is an ugly O(n^2) hack (XXX).
   *)
  let csv = List.filter (
    fun row -> not (List.exists (equal row) update)
  ) csv in
  let csv = csv @ update in
  Csv.output_all (Csv.to_channel ~separator:output_sep chan) csv

let cmd_call ~input_sep ~output_sep ~chan command files =
  (* Avoid loading the whole file into memory. *)
  let f row =
    let cmd =
      command ^ " " ^ String.concat " " (List.map Filename.quote row) in
    let code = Sys.command cmd in
    if code <> 0 then (
      eprintf "%s: terminated with exit code %d\n" command code;
      exit code
    )
  in
  List.iter (
    fun filename ->
      let in_chan, close =
        match filename with
        | "-" -> stdin, false
        | filename -> open_in filename, true in
      Csv.iter f (Csv.of_channel ~separator:input_sep in_chan);
      if close then close_in in_chan
  ) files

let rec uniq = function
  | [] -> []
  | [x] -> [x]
  | x :: y :: xs when Pervasives.compare x y = 0 ->
      uniq (x :: xs)
  | x :: y :: xs ->
      x :: uniq (y :: xs)

let cmd_join ~input_sep ~output_sep ~chan colspec1 colspec2 files =
  (* Load in the files separately. *)
  let csvs = List.map (Csv.load ~separator:input_sep) files in

  (* For each CSV file, construct a hash table from row class (key) to
   * the (possibly empty) output columns (values).
   * Also construct a hash which has the unique list of row classes.
   *)
  let keys = Hashtbl.create 1023 in
  let hashes = List.map (
    fun csv ->
      let hash = Hashtbl.create 1023 in
      List.iter (
        fun row ->
          let key = cols_of_colspec colspec1 row in
          let value = cols_of_colspec colspec2 row in
          if not (Hashtbl.mem keys key) then Hashtbl.add keys key true;
          Hashtbl.add hash key value
      ) csv;
      hash
  ) csvs in

  (* Get the keys. *)
  let keys = Hashtbl.fold (fun key _ xs -> key :: xs) keys [] in

  let value_width = width_of_colspec colspec2 in
  let empty_value =
    List.hd (Csv.set_columns value_width [[""]]) in
  let multiple_values =
    List.hd (Csv.set_columns value_width [["!MULTIPLE VALUES"]]) in

  (* Generate output CSV. *)
  let keys = List.sort Pervasives.compare keys in
  let keys = List.map (fun key -> key, []) keys in
  let csv = List.fold_left (
    fun keys hash ->
      List.map (
        fun (key, values) ->
          let value = try Hashtbl.find_all hash key with Not_found -> [] in
          let value =
            match value with
            | [] -> empty_value
            | [value] -> value
            | _::_ -> multiple_values in
          key, (value :: values)
      ) keys
  ) keys hashes in
  let csv = List.map (
    fun (key, values) ->
      key @ List.flatten (List.rev values)
  ) csv in
  Csv.output_all (Csv.to_channel ~separator:output_sep chan) csv

let rec cmd_trim ~input_sep ~output_sep ~chan (top, left, right, bottom) files =
  let csv = List.concat (List.map (Csv.load ~separator:input_sep) files) in
  let csv = Csv.trim ~top ~left ~right ~bottom csv in
  Csv.output_all (Csv.to_channel ~separator:output_sep chan) csv
{'fraction_non_alphanumeric': 0.10076816847919903, 'fraction_numerical': 0.004833419644398412, 'mean_word_length': 2.2086679590141234, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 145, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '875145', 'n_tokens_mistral': 7812, 'n_tokens_neox': 7490, 'n_words': 2801}