<commit_before>let is_dir_sep =
  if Sys.win32 || Sys.cygwin then
    fun c -> c = '/' || c = '\\' || c = ':'
  else
    fun c -> c = '/'

let explode_path =
  let rec start acc path i =
    if i < 0 then
      acc
    else if is_dir_sep (String.unsafe_get path i) then
      start acc path (i - 1)
    else
      component acc path i (i - 1)
  and component acc path end_ i =
    if i < 0 then
      String.take path (end_ + 1) :: acc
    else if is_dir_sep (String.unsafe_get path i) then
      start
        (String.sub path ~pos:(i + 1) ~len:(end_ - i)::acc)
        path
        (i - 1)
    else
      component acc path end_ (i - 1)
  in
  fun path ->
    if path = Filename.current_dir_name then
      [path]
    else
      match start [] path (String.length path - 1) with
      | "." :: xs -> xs
      | xs -> xs

module External : sig
  include Path_intf.S
  val compare_val : t -> t -> Ordering.t
  val relative : t -> string -> t
  val mkdir_p : t -> unit
  val initial_cwd : t
  val cwd : unit -> t
  val as_local : t -> string
end = struct
  include Interned.No_interning(struct
      let initial_size = 512
      let resize_policy = Interned.Greedy
      let order = Interned.Natural
    end)()

  let compare_val x y = String.compare (to_string x) (to_string y)

  let as_string x ~f =
    to_string x
    |> f
    |> make

  let extend_basename t ~suffix = as_string t ~f:(fun t -> t ^ suffix)

  let is_suffix t ~suffix = String.is_suffix (to_string t) ~suffix

  let of_string t =
    if Filename.is_relative t then
      Exn.code_error "Path.External.of_string: relative path given"
        [ "t", Sexp.Encoder.string t ];
    make t

  let parse_string_exn ~loc t =
    if Filename.is_relative t then
      Exn.fatalf ~loc "path %s is not absolute" t;
    make t

  let to_sexp t = Sexp.Encoder.string (to_string t)
  let to_dyn t = Dyn.String (to_string t)

(*
  let rec cd_dot_dot t =
    match Unix.readlink t with
    | exception _ -> Filename.dirname t
    | t -> cd_dot_dot t

  let relative initial_t path =
    let rec loop t components =
      match components with
      | [] | ["." | ".."] ->
        die "invalid filename concatenation: %s / %s" initial_t path
      | [fn] -> Filename.concat t fn
      | "."  :: rest -> loop t rest
      | ".." :: rest -> loop (cd_dot_dot t) rest
      | comp :: rest -> loop (Filename.concat t comp) rest
    in
    loop initial_t (explode_path path)
*)

  let relative x y =
    match y with
    | "." -> x
    | _   -> make (Filename.concat (to_string x) y)

  let rec mkdir_p t =
    let t_s = to_string t in
    let p_s = Filename.dirname t_s in
    let p = make p_s in
    if p <> t then
      try
        Unix.mkdir t_s 0o777
      with
      | Unix.Unix_error (EEXIST, _, _) -> ()
      | Unix.Unix_error (ENOENT, _, _) ->
        mkdir_p p;
        Unix.mkdir t_s 0o777

  let basename t = Filename.basename (to_string t)
  let parent t =
    match Filename.dirname (to_string t) with
    | "." -> None
    | s -> Some (make s)

  let extension t = Filename.extension (to_string t)
  let split_extension t =
    let s, ext = Filename.split_extension (to_string t) in
    (make s, ext)

  let set_extension t ~ext =
    let (base, _) = split_extension t in
    (to_string base) ^ ext
    |> make

  let cwd () = make (Sys.getcwd ())
  let initial_cwd = cwd ()

  let as_local t =
    let s = to_string t in
    "." ^ s

  include (
    Comparable.Operators(struct
      type nonrec t = t
      let compare = compare_val
    end)
    : Comparable.OPS with type t := t
  )

  let to_string_maybe_quoted t =
    String.maybe_quoted (to_string t)

  let parent_exn t =
    match parent t with
    | None -> Exn.code_error "Path.External.parent_exn called on a root path" []
    | Some p -> p

  let root = of_string "/"

  let is_root = equal root

  let is_descendant b ~of_:a =
    if is_root a then true
    else
      String.is_prefix ~prefix:(to_string a ^ "/") (to_string b)

end

module Relative : sig
  include Path_intf.S

  val root : t
  val is_root : t -> bool
  val compare_val : t -> t -> Ordering.t
  val relative : ?error_loc:Loc0.t -> t -> string -> t
  val append : t -> t -> t
  val mkdir_p : t -> unit
  val descendant : t -> of_:t -> t option
  val is_descendant : t -> of_:t -> bool
  val reach : t -> from:t -> string

  module L : sig
    val relative : ?error_loc:Loc0.t -> t -> string list -> t
  end<commit_msg>Fix External.parent (#2153)<commit_after>let is_dir_sep =
  if Sys.win32 || Sys.cygwin then
    fun c -> c = '/' || c = '\\' || c = ':'
  else
    fun c -> c = '/'

let explode_path =
  let rec start acc path i =
    if i < 0 then
      acc
    else if is_dir_sep (String.unsafe_get path i) then
      start acc path (i - 1)
    else
      component acc path i (i - 1)
  and component acc path end_ i =
    if i < 0 then
      String.take path (end_ + 1) :: acc
    else if is_dir_sep (String.unsafe_get path i) then
      start
        (String.sub path ~pos:(i + 1) ~len:(end_ - i)::acc)
        path
        (i - 1)
    else
      component acc path end_ (i - 1)
  in
  fun path ->
    if path = Filename.current_dir_name then
      [path]
    else
      match start [] path (String.length path - 1) with
      | "." :: xs -> xs
      | xs -> xs

module External : sig
  include Path_intf.S
  val compare_val : t -> t -> Ordering.t
  val relative : t -> string -> t
  val mkdir_p : t -> unit
  val initial_cwd : t
  val cwd : unit -> t
  val as_local : t -> string
end = struct
  include Interned.No_interning(struct
      let initial_size = 512
      let resize_policy = Interned.Greedy
      let order = Interned.Natural
    end)()

  let compare_val x y = String.compare (to_string x) (to_string y)

  let as_string x ~f =
    to_string x
    |> f
    |> make

  let extend_basename t ~suffix = as_string t ~f:(fun t -> t ^ suffix)

  let is_suffix t ~suffix = String.is_suffix (to_string t) ~suffix

  let of_string t =
    if Filename.is_relative t then
      Exn.code_error "Path.External.of_string: relative path given"
        [ "t", Sexp.Encoder.string t ];
    make t

  let parse_string_exn ~loc t =
    if Filename.is_relative t then
      Exn.fatalf ~loc "path %s is not absolute" t;
    make t

  let to_sexp t = Sexp.Encoder.string (to_string t)
  let to_dyn t = Dyn.String (to_string t)

(*
  let rec cd_dot_dot t =
    match Unix.readlink t with
    | exception _ -> Filename.dirname t
    | t -> cd_dot_dot t

  let relative initial_t path =
    let rec loop t components =
      match components with
      | [] | ["." | ".."] ->
        die "invalid filename concatenation: %s / %s" initial_t path
      | [fn] -> Filename.concat t fn
      | "."  :: rest -> loop t rest
      | ".." :: rest -> loop (cd_dot_dot t) rest
      | comp :: rest -> loop (Filename.concat t comp) rest
    in
    loop initial_t (explode_path path)
*)

  let relative x y =
    match y with
    | "." -> x
    | _   -> make (Filename.concat (to_string x) y)

  let rec mkdir_p t =
    let t_s = to_string t in
    let p_s = Filename.dirname t_s in
    let p = make p_s in
    if p <> t then
      try
        Unix.mkdir t_s 0o777
      with
      | Unix.Unix_error (EEXIST, _, _) -> ()
      | Unix.Unix_error (ENOENT, _, _) ->
        mkdir_p p;
        Unix.mkdir t_s 0o777

  let basename t = Filename.basename (to_string t)

  let root = of_string "/"

  let is_root = equal root

  let parent t =
    if is_root t then
      None
    else
      Some (as_string t ~f:Filename.dirname)

  let extension t = Filename.extension (to_string t)
  let split_extension t =
    let s, ext = Filename.split_extension (to_string t) in
    (make s, ext)

  let set_extension t ~ext =
    let (base, _) = split_extension t in
    (to_string base) ^ ext
    |> make

  let cwd () = make (Sys.getcwd ())
  let initial_cwd = cwd ()

  let as_local t =
    let s = to_string t in
    "." ^ s

  include (
    Comparable.Operators(struct
      type nonrec t = t
      let compare = compare_val
    end)
    : Comparable.OPS with type t := t
  )

  let to_string_maybe_quoted t =
    String.maybe_quoted (to_string t)

  let parent_exn t =
    match parent t with
    | None -> Exn.code_error "Path.External.parent_exn called on a root path" []
    | Some p -> p

  let is_descendant b ~of_:a =
    if is_root a then true
    else
      String.is_prefix ~prefix:(to_string a ^ "/") (to_string b)

end

module Relative : sig
  include Path_intf.S

  val root : t
  val is_root : t -> bool
  val compare_val : t -> t -> Ordering.t
  val relative : ?error_loc:Loc0.t -> t -> string -> t
  val append : t -> t -> t
  val mkdir_p : t -> unit
  val descendant : t -> of_:t -> t option
  val is_descendant : t -> of_:t -> bool
  val reach : t -> from:t -> string

  module L : sig
    val relative : ?error_loc:Loc0.t -> t -> string list -> t
  end
{'fraction_non_alphanumeric': 0.10741369552914544, 'fraction_numerical': 0.00588568194680249, 'mean_word_length': 2.1145576313006695, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '764175', 'n_tokens_mistral': 3275, 'n_tokens_neox': 3089, 'n_words': 1059}