<commit_before>native "rust" mod rustrt {
  fn rust_file_is_dir(str path) -> int;
}

fn path_sep() -> str {
    ret _str.from_char(os_fs.path_sep);
}

type path = str;

fn dirname(path p) -> path {
    auto sep = path_sep();
    assert (_str.byte_len(sep) == 1u);
    let int i = _str.rindex(p, sep.(0));
    if (i == -1) {
        ret p;
    }
    ret _str.substr(p, 0u, i as uint);
}

fn connect(path pre, path post) -> path {
    auto len = _str.byte_len(pre);
    if (pre.(len - 1u) == (os_fs.path_sep as u8)) { // Trailing '/'?
        ret pre + post;
    }
    ret pre + path_sep() + post;
}

fn file_is_dir(path p) -> bool {
  ret rustrt.rust_file_is_dir(p) != 0;
}

fn list_dir(path p) -> vec[str] {
  auto pl = _str.byte_len(p);
  if (pl == 0u || p.(pl - 1u) as char != os_fs.path_sep) {
    p += path_sep();
  }<commit_msg>Fix assumption that only os_fs.path_sep can separate paths, sigh.<commit_after>native "rust" mod rustrt {
  fn rust_file_is_dir(str path) -> int;
}

fn path_sep() -> str {
    ret _str.from_char(os_fs.path_sep);
}

type path = str;

fn dirname(path p) -> path {
    auto sep = path_sep();
    assert (_str.byte_len(sep) == 1u);
    let int i = _str.rindex(p, sep.(0));
    if (i == -1) {
        // FIXME: the '/' character is a path separator on all 3 platforms we
        // support. This should probably be generalized a bit more in the
        // future, but for now this should work.
        i = _str.rindex(p, '/');
        if (i == -1) {
            ret p;
        }
    }
    ret _str.substr(p, 0u, i as uint);
}

fn connect(path pre, path post) -> path {
    auto len = _str.byte_len(pre);
    if (pre.(len - 1u) == (os_fs.path_sep as u8)) { // Trailing '/'?
        ret pre + post;
    }
    ret pre + path_sep() + post;
}

fn file_is_dir(path p) -> bool {
  ret rustrt.rust_file_is_dir(p) != 0;
}

fn list_dir(path p) -> vec[str] {
  auto pl = _str.byte_len(p);
  if (pl == 0u || p.(pl - 1u) as char != os_fs.path_sep) {
    p += path_sep();
  }
{'fraction_non_alphanumeric': 0.1649069884364002, 'fraction_numerical': 0.010055304172951232, 'mean_word_length': 2.3671742808798646, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3459327', 'n_tokens_mistral': 807, 'n_tokens_neox': 789, 'n_words': 221}