<commit_before>open Types
open Printf
open Common

(** [ensure cmd] Returns [true] if a given [command] is available
    on the host system and [false] otherwise. *)
let ensure cmd =
  Sys.command (sprintf "sh -c 'which %s &> /dev/null'" cmd) = 0


class remote url : source_type = object
  method is_available () = List.for_all ensure ["wget"]

  method fetch ~dest_dir =
    let file_path = dest_dir </> Filename.basename url in
    let open Res in
        let command fmt = Printf.ksprintf Sys.command_ok fmt in
        let res = if Sys.file_exists file_path then
            return ()
          else
            command "wget --no-check-certificate %s -O %s" url file_path
        in res >>= fun () -> return file_path
end


class archive archive_type file_path : source_type = object
  method is_available () = List.for_all ensure ["tar"]

  method fetch ~dest_dir =
    let archive_cmd = match archive_type with
      | `Tar -> "tar -xf "
      | `TarGz -> "tar -zxf"
      | `TarBzip2 -> "tar -jxf"
    in

    let open Res in
        let command fmt = Printf.ksprintf Sys.command_ok fmt in
        command "mkdir -p %s" dest_dir >>= fun () ->
        command "%s %s -C %s" archive_cmd file_path dest_dir >>= fun () ->
        return dest_dir
end<commit_msg>Changed 'Source.archive' to return a proper source dir<commit_after>open Types
open Printf
open Common

(** [ensure cmd] Returns [true] if a given [command] is available
    on the host system and [false] otherwise. *)
let ensure cmd =
  Sys.command (sprintf "sh -c 'which %s &> /dev/null'" cmd) = 0


class remote url : source_type = object
  method is_available () = List.for_all ensure ["wget"]

  method fetch ~dest_dir =
    let file_path = dest_dir </> Filename.basename url in
    let open Res in
        let command fmt = Printf.ksprintf Sys.command_ok fmt in
        let res = if Sys.file_exists file_path then
            return ()
          else
            command "wget --no-check-certificate %s -O %s" url file_path
        in res >>= fun () -> return file_path
end


class archive archive_type file_path : source_type = object
  method is_available () = List.for_all ensure ["tar"]

  method fetch ~dest_dir =
    let archive_cmd = match archive_type with
      | `Tar -> "tar -xf "
      | `TarGz -> "tar -zxf"
      | `TarBzip2 -> "tar -jxf"
    in

    let open Res in
        let command fmt = Printf.ksprintf Sys.command_ok fmt in
        command "mkdir -p %s" dest_dir >>= fun () ->
        command "%s %s -C %s" archive_cmd file_path dest_dir >>= fun () ->

        (* If [dest_dir] contains a single directory, assume it *is* the
           source dir, otherwise return [dest_dir]. *)
        let files = Array.map ((</>) dest_dir) (Sys.readdir dest_dir)
        in match List.filter Sys.is_directory (Array.to_list files) with
          | [d] -> return d
          | _   -> return dest_dir
end
{'fraction_non_alphanumeric': 0.1065232477446218, 'fraction_numerical': 0.0013879250520471894, 'mean_word_length': 2.465144230769231, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3894893', 'n_tokens_mistral': 925, 'n_tokens_neox': 900, 'n_words': 327}