<commit_before>(*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 *)

(** Source code debug locations *)

type t = {dir: string; file: string; line: int; col: int}
[@@deriving compare, equal, hash, sexp]

let none = {dir= ""; file= ""; line= 0; col= 0}

let mk ?(dir = none.dir) ?(file = none.file) ?(col = none.col) ~line =
  let dir = if String.is_empty dir then dir else Filename.realpath dir in
  {dir; file; line; col}

let root = ref (Filename.realpath (Sys.getcwd ()))

let pp fs ({dir; file; line; col} as loc) =
  if not (equal loc none) then Format.pp_print_string fs "; " ;
  if not (String.is_empty dir) then (
    let dir =
      Option.map_or ~f:Fpath.to_string
        (Fpath.relativize ~root:(Fpath.v !root) (Fpath.v dir))
        ~default:dir
    in
    Format.pp_print_string fs dir ;
    Format.pp_print_string fs Filename.dir_sep ) ;
  Format.pp_print_string fs file ;
  if not (String.is_empty file) then Format.pp_print_char fs ':' ;
  if line > 0 then (
    Format.pp_print_int fs line ;
    Format.pp_print_char fs ':' ) ;
  if col > 0 then (
    Format.pp_print_int fs col ;
    Format.pp_print_char fs ':' )

let pp fs l = Format.pp_print_as fs 0 (Format.asprintf "%a" pp l)<commit_msg>[sledge] Protect against nonexistent paths in debug info<commit_after>(*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 *)

(** Source code debug locations *)

type t = {dir: string; file: string; line: int; col: int}
[@@deriving compare, equal, hash, sexp]

let none = {dir= ""; file= ""; line= 0; col= 0}

let mk ?(dir = none.dir) ?(file = none.file) ?(col = none.col) ~line =
  let dir =
    if String.is_empty dir then dir
    else try Filename.realpath dir with Unix.Unix_error _ -> dir
  in
  {dir; file; line; col}

let root = ref (Filename.realpath (Sys.getcwd ()))

let pp fs ({dir; file; line; col} as loc) =
  if not (equal loc none) then Format.pp_print_string fs "; " ;
  if not (String.is_empty dir) then (
    let dir =
      Option.map_or ~f:Fpath.to_string
        (Fpath.relativize ~root:(Fpath.v !root) (Fpath.v dir))
        ~default:dir
    in
    Format.pp_print_string fs dir ;
    Format.pp_print_string fs Filename.dir_sep ) ;
  Format.pp_print_string fs file ;
  if not (String.is_empty file) then Format.pp_print_char fs ':' ;
  if line > 0 then (
    Format.pp_print_int fs line ;
    Format.pp_print_char fs ':' ) ;
  if col > 0 then (
    Format.pp_print_int fs col ;
    Format.pp_print_char fs ':' )

let pp fs l = Format.pp_print_as fs 0 (Format.asprintf "%a" pp l)
{'fraction_non_alphanumeric': 0.12270600935588341, 'fraction_numerical': 0.0035984166966534724, 'mean_word_length': 3.2249240121580547, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2451594', 'n_tokens_mistral': 986, 'n_tokens_neox': 966, 'n_words': 338}