<commit_before>(*
 * Copyright (C) 2006-2009 Citrix Systems Inc.
 * Copyright (C)      2012 <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; version 2.1 only. with the special
 * exception on linking described in file LICENSE.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *)

module String = struct
  include String
  open String

  let of_char c = String.make 1 c

  let fold_right f string accu =
    let accu = ref accu in
    for i = length string - 1 downto 0 do
      accu := f string.[i] !accu
    done;
    !accu

  let explode string =
    fold_right (fun h t -> h :: t) string []

  let implode list =
    concat "" (List.map of_char list)

end

let rec really_read fd string off n =
  if n=0 then () else
    let m = Unix.read fd string off n in
    if m = 0 then raise End_of_file;
    really_read fd string (off+m) (n-m)

let finally fct clean_f =
  let result =
    try fct ();
    with exn ->
      clean_f ();
      raise exn in
  clean_f ();
  result

(** Process and create tar file headers *)
module Header = struct
  (** Map of field name -> (start offset, length) taken from wikipedia:
      http://en.wikipedia.org/w/index.php?title=Tar_%28file_format%29&oldid=83554041 *)

  cstruct hdr {
    uint8_t file_name[100];
    uint8_t file_mode[8];
    uint8_t user_id[8];
    uint8_t group_id[8];
    uint8_t file_size[12];
    uint8_t mod_time[12];
    uint8_t chksum[8];
    uint8_t link_indicator;
    uint8_t link_name[100]
  } as little_endian (* doesn't matter, all are strings *)

  let sizeof_hdr_file_name = 100
  let sizeof_hdr_file_mode = 8
  let sizeof_hdr_user_id   = 8
  let sizeof_hdr_group_id  = 8
  let sizeof_hdr_file_size = 12
  let sizeof_hdr_mod_time  = 12
  let sizeof_hdr_chksum    = 8
  let sizeof_hdr_link_name = 100

  module Link = struct
    type t =
      | Normal
      | Hard
      | Symbolic

    let to_int = function
      | Normal   -> 0
      | Hard     -> 49 (* '1' *)
      | Symbolic -> 50 (* '2' *)

    let of_int = function
      | 49 (* '1' *) -> Hard
      | 50 (* '2' *) -> Symbolic
      | _ -> Normal (* if value is malformed, treat as a normal file *)

    let to_string = function
      | Normal -> "Normal"
      | Hard -> "Hard"
      | Symbolic -> "Symbolic"
  end 

  (** Represents a standard (non-USTAR) archive (note checksum not stored) *)
  type t = { file_name: string;
	     file_mode: int;
	     user_id: int;
	     group_id: int;
	     file_size: int64;
	     mod_time: int64;
	     link_indicator: Link.t;
	     link_name: string;
	   }

  (** Helper function to make a simple header *)
  let make ?(file_mode=0) ?(user_id=0) ?(group_id=0) ?(mod_time=0L) ?(link_indicator=Link.Normal) ?(link_name="") file_name file_size = 
    { file_name = file_name;
      file_mode = file_mode;
      user_id = user_id;
      group_id = group_id;
      file_size = file_size;
      mod_time = mod_time;
      link_indicator;
      link_name }

  (** Length of a header block *)
  let length = 512

  (** A blank header block (two of these in series mark the end of the tar) *)
  let zero_block = String.make length '\000'

  (** Return a string containing 'x' padded out to 'n' bytes by adding 'c' to the LHS *)
  let pad_left (x: string) (n: int) (c: char) = 
    if String.length x >= n then x
    else let buffer = String.make n c in
         String.blit x 0 buffer (n - (String.length x)) (String.length x);
         buffer

  (** Return a string containing 'x' padded out to 'n' bytes by adding 'c' to the RHS *)
  let pad_right (x: string) (n: int) (c: char) = 
    if String.length x >= n then x
    else let buffer = String.make n c in
         String.blit x 0 buffer 0 (String.length x);
         buffer

  (** Pretty-print the header record *)
  let to_detailed_string (x: t) = 
    let table = [ "file_name",      x.file_name;
		  "file_mode",      string_of_int x.file_mode;
		  "user_id",        string_of_int x.user_id;
		  "group_id",       string_of_int x.group_id;
		  "file_size",      Int64.to_string x.file_size;
		  "mod_time",       Int64.to_string x.mod_time;
		  "link_indicator", Link.to_string x.link_indicator;
		  "link_name",      x.link_name ] in
    "{\n" ^ (String.concat "\n\t" (List.map (fun (k, v) -> k ^ ": " ^ v) table)) ^ "}"

  (** Make a single line summary which looks like the output of tar -tv *)
  let to_summary_string (x: t) = 
    (* -rw-r--r-- *)
    let mode = Printf.sprintf "%010d" x.file_mode in
    (* root/root *)
    let usergroup = Printf.sprintf "%d/%d" x.user_id x.group_id in
    let size = pad_right (Int64.to_string x.file_size) 8 ' ' in
    let time = Unix.gmtime (Int64.to_float x.mod_time) in
    let time = Printf.sprintf "%04d-%02d-%02d %02d:%02d:%02d" 
      (time.Unix.tm_year + 1900) (time.Unix.tm_mon + 1) time.Unix.tm_mday
      time.Unix.tm_hour time.Unix.tm_min time.Unix.tm_sec in
    Printf.sprintf "%s %s %s %s %s" mode usergroup size time x.file_name

  (** For debugging: pretty-print a string as hex *)
  let to_hex (x: string) : string = 
    let chars = List.map (Printf.sprintf "%02x") (List.map int_of_char (String.explode x)) in
    String.concat " " chars

  (** Marshal an integer field of size 'n' *)
  let marshal_int (x: int) (n: int) = 
    let octal = Printf.sprintf "%o" x in
    let result = pad_left octal (n-1) '0' in
    result ^ "\000" (* space or NULL allowed *)

  (** Marshal an int64 field of size 'n' *)
  let marshal_int64 (x: int64) (n: int) = 
    let octal = Printf.sprintf "%Lo" x in
    let result = pad_left octal (n-1) '0' in
    result ^ "\000" (* space or NULL allowed *)

  (** Marshal an string field of size 'n' *)
  let marshal_string (x: string) (n: int) = pad_right x n '\000'

  let trim regexp x = match Re_str.split regexp x with
    | [] -> ""
    | x :: _ -> x
  let trim_numerical = trim (Re_str.regexp "[\000 ]+")
  let trim_string = trim (Re_str.regexp "[\000]+")

  (** Unmarshal an integer field (stored as 0-padded octal) *)
  let unmarshal_int (x: string) : int = <commit_msg>tar: remove overload of standard library String module<commit_after>(*
 * Copyright (C) 2006-2009 Citrix Systems Inc.
 * Copyright (C)      2012 <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; version 2.1 only. with the special
 * exception on linking described in file LICENSE.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *)

let rec really_read fd string off n =
  if n=0 then () else
    let m = Unix.read fd string off n in
    if m = 0 then raise End_of_file;
    really_read fd string (off+m) (n-m)

let finally fct clean_f =
  let result =
    try fct ();
    with exn ->
      clean_f ();
      raise exn in
  clean_f ();
  result

(** Process and create tar file headers *)
module Header = struct
  (** Map of field name -> (start offset, length) taken from wikipedia:
      http://en.wikipedia.org/w/index.php?title=Tar_%28file_format%29&oldid=83554041 *)

  cstruct hdr {
    uint8_t file_name[100];
    uint8_t file_mode[8];
    uint8_t user_id[8];
    uint8_t group_id[8];
    uint8_t file_size[12];
    uint8_t mod_time[12];
    uint8_t chksum[8];
    uint8_t link_indicator;
    uint8_t link_name[100]
  } as little_endian (* doesn't matter, all are strings *)

  let sizeof_hdr_file_name = 100
  let sizeof_hdr_file_mode = 8
  let sizeof_hdr_user_id   = 8
  let sizeof_hdr_group_id  = 8
  let sizeof_hdr_file_size = 12
  let sizeof_hdr_mod_time  = 12
  let sizeof_hdr_chksum    = 8
  let sizeof_hdr_link_name = 100

  module Link = struct
    type t =
      | Normal
      | Hard
      | Symbolic

    let to_int = function
      | Normal   -> 0
      | Hard     -> 49 (* '1' *)
      | Symbolic -> 50 (* '2' *)

    let of_int = function
      | 49 (* '1' *) -> Hard
      | 50 (* '2' *) -> Symbolic
      | _ -> Normal (* if value is malformed, treat as a normal file *)

    let to_string = function
      | Normal -> "Normal"
      | Hard -> "Hard"
      | Symbolic -> "Symbolic"
  end 

  (** Represents a standard (non-USTAR) archive (note checksum not stored) *)
  type t = { file_name: string;
	     file_mode: int;
	     user_id: int;
	     group_id: int;
	     file_size: int64;
	     mod_time: int64;
	     link_indicator: Link.t;
	     link_name: string;
	   }

  (** Helper function to make a simple header *)
  let make ?(file_mode=0) ?(user_id=0) ?(group_id=0) ?(mod_time=0L) ?(link_indicator=Link.Normal) ?(link_name="") file_name file_size = 
    { file_name = file_name;
      file_mode = file_mode;
      user_id = user_id;
      group_id = group_id;
      file_size = file_size;
      mod_time = mod_time;
      link_indicator;
      link_name }

  (** Length of a header block *)
  let length = 512

  (** A blank header block (two of these in series mark the end of the tar) *)
  let zero_block = String.make length '\000'

  (** Return a string containing 'x' padded out to 'n' bytes by adding 'c' to the LHS *)
  let pad_left (x: string) (n: int) (c: char) = 
    if String.length x >= n then x
    else let buffer = String.make n c in
         String.blit x 0 buffer (n - (String.length x)) (String.length x);
         buffer

  (** Return a string containing 'x' padded out to 'n' bytes by adding 'c' to the RHS *)
  let pad_right (x: string) (n: int) (c: char) = 
    if String.length x >= n then x
    else let buffer = String.make n c in
         String.blit x 0 buffer 0 (String.length x);
         buffer

  (** Pretty-print the header record *)
  let to_detailed_string (x: t) = 
    let table = [ "file_name",      x.file_name;
		  "file_mode",      string_of_int x.file_mode;
		  "user_id",        string_of_int x.user_id;
		  "group_id",       string_of_int x.group_id;
		  "file_size",      Int64.to_string x.file_size;
		  "mod_time",       Int64.to_string x.mod_time;
		  "link_indicator", Link.to_string x.link_indicator;
		  "link_name",      x.link_name ] in
    "{\n" ^ (String.concat "\n\t" (List.map (fun (k, v) -> k ^ ": " ^ v) table)) ^ "}"

  (** Make a single line summary which looks like the output of tar -tv *)
  let to_summary_string (x: t) = 
    (* -rw-r--r-- *)
    let mode = Printf.sprintf "%010d" x.file_mode in
    (* root/root *)
    let usergroup = Printf.sprintf "%d/%d" x.user_id x.group_id in
    let size = pad_right (Int64.to_string x.file_size) 8 ' ' in
    let time = Unix.gmtime (Int64.to_float x.mod_time) in
    let time = Printf.sprintf "%04d-%02d-%02d %02d:%02d:%02d" 
      (time.Unix.tm_year + 1900) (time.Unix.tm_mon + 1) time.Unix.tm_mday
      time.Unix.tm_hour time.Unix.tm_min time.Unix.tm_sec in
    Printf.sprintf "%s %s %s %s %s" mode usergroup size time x.file_name

  (** For debugging: pretty-print a string as hex *)
  let to_hex (x: string) : string =
    let result = String.make (String.length x * 3) ' ' in
    for i = 0 to String.length x - 1 do
      let byte = Printf.sprintf "%02x" (int_of_char x.[i]) in
      String.blit byte 0 result (i * 3) 2
    done;
    result

  (** Marshal an integer field of size 'n' *)
  let marshal_int (x: int) (n: int) = 
    let octal = Printf.sprintf "%o" x in
    let result = pad_left octal (n-1) '0' in
    result ^ "\000" (* space or NULL allowed *)

  (** Marshal an int64 field of size 'n' *)
  let marshal_int64 (x: int64) (n: int) = 
    let octal = Printf.sprintf "%Lo" x in
    let result = pad_left octal (n-1) '0' in
    result ^ "\000" (* space or NULL allowed *)

  (** Marshal an string field of size 'n' *)
  let marshal_string (x: string) (n: int) = pad_right x n '\000'

  let trim regexp x = match Re_str.split regexp x with
    | [] -> ""
    | x :: _ -> x
  let trim_numerical = trim (Re_str.regexp "[\000 ]+")
  let trim_string = trim (Re_str.regexp "[\000]+")

  (** Unmarshal an integer field (stored as 0-padded octal) *)
  let unmarshal_int (x: string) : int = 
{'fraction_non_alphanumeric': 0.11765651370248333, 'fraction_numerical': 0.025154705456883388, 'mean_word_length': 2.664310954063604, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6661352', 'n_tokens_mistral': 4610, 'n_tokens_neox': 4316, 'n_words': 1457}