<commit_before>
import front::ast;
import front::ast::ident;
import front::ast::node_id;
import front::ast::def_id;
import util::common::span;
import visit::vt;
import std::vec;
import std::str;
import std::option;
import std::option::some;
import std::option::none;
import std::option::is_none;


// This is not an alias-analyser (though it would merit from becoming one, or
// at getting input from one, to be more precise). It is a pass that checks
// whether aliases are used in a safe way. Beyond that, though it doesn't have
// a lot to do with aliases, it also checks whether assignments are valid
// (using an lval, which is actually mutable), since it already has all the
// information needed to do that (and the typechecker, which would be a
// logical place for such a check, doesn't).
tag valid { valid; overwritten(span, ast::path); val_taken(span, ast::path); }

type restrict =
    @rec(vec[node_id] root_vars,
         node_id block_defnum,
         vec[node_id] bindings,
         vec[ty::t] tys,
         vec[uint] depends_on,
         mutable valid ok);

type scope = vec[restrict];

tag local_info { arg(ast::mode); objfield(ast::mutability); }

type ctx = rec(@ty::ctxt tcx,
               std::map::hashmap[node_id, local_info] local_map);

fn check_crate(@ty::ctxt tcx, &@ast::crate crate) {
    auto cx =
        @rec(tcx=tcx,

             // Stores information about object fields and function
             // arguments that's otherwise not easily available.
             local_map=util::common::new_int_hash());
    auto v =
        @rec(visit_fn=bind visit_fn(cx, _, _, _, _, _, _, _),
             visit_item=bind visit_item(cx, _, _, _),
             visit_expr=bind visit_expr(cx, _, _, _)
             with *visit::default_visitor[scope]());
    visit::visit_crate(*crate, [], visit::vtor(v));
}

fn visit_fn(@ctx cx, &ast::_fn f, &vec[ast::ty_param] tp, &span sp,
            &ident name, ast::node_id id, &scope sc, &vt[scope] v) {
    visit::visit_fn_decl(f.decl, sc, v);
    for (ast::arg arg_ in f.decl.inputs) {
        cx.local_map.insert(arg_.id, arg(arg_.mode));
    }
    vt(v).visit_block(f.body, [], v);
}

fn visit_item(@ctx cx, &@ast::item i, &scope sc, &vt[scope] v) {
    alt (i.node) {
        case (ast::item_obj(?o, _, _)) {
            for (ast::obj_field f in o.fields) {
                cx.local_map.insert(f.id, objfield(f.mut));
            }
        }
        case (_) { }
    }
    visit::visit_item(i, sc, v);
}

fn visit_expr(@ctx cx, &@ast::expr ex, &scope sc, &vt[scope] v) {
    auto handled = true;
    alt (ex.node) {
        case (ast::expr_call(?f, ?args, _)) {
            check_call(*cx, f, args, sc);
            handled = false;
        }
        case (ast::expr_be(?cl, _)) {
            check_tail_call(*cx, cl);
            visit::visit_expr(cl, sc, v);
        }
        case (ast::expr_alt(?input, ?arms, _)) {
            check_alt(*cx, input, arms, sc, v);
        }
        case (ast::expr_put(?val, _)) {
            alt (val) {
                case (some(?ex)) {
                    auto root = expr_root(*cx, ex, false);
                    if (mut_field(root.ds)) {
                        cx.tcx.sess.span_fatal(ex.span,
                                             "result of put must be" +
                                                 " immutably rooted");
                    }
                    visit_expr(cx, ex, sc, v);
                }
                case (_) { }
            }
        }
        case (ast::expr_for_each(?decl, ?call, ?block, _)) {
            check_for_each(*cx, decl, call, block, sc, v);
        }
        case (ast::expr_for(?decl, ?seq, ?block, _)) {
            check_for(*cx, decl, seq, block, sc, v);
        }
        case (ast::expr_path(?pt, ?id)) {
            check_var(*cx, ex, pt, id, false, sc);
            handled = false;
        }
        case (ast::expr_move(?dest, ?src, _)) {
            check_assign(cx, dest, src, sc, v);
        }
        case (ast::expr_assign(?dest, ?src, _)) {
            check_assign(cx, dest, src, sc, v);
        }
        case (ast::expr_assign_op(_, ?dest, ?src, _)) {
            check_assign(cx, dest, src, sc, v);
        }
        case (_) { handled = false; }
    }
    if (!handled) { visit::visit_expr(ex, sc, v); }
}

fn check_call(&ctx cx, &@ast::expr f, &vec[@ast::expr] args, &scope sc) ->
   rec(vec[node_id] root_vars, vec[ty::t] unsafe_ts) {
    auto fty = ty::expr_ty(*cx.tcx, f);
    auto arg_ts = fty_args(cx, fty);
    let vec[node_id] roots = [];
    let vec[tup(uint, node_id)] mut_roots = [];
    let vec[ty::t] unsafe_ts = [];
    let vec[uint] unsafe_t_offsets = [];
    auto i = 0u;
    for (ty::arg arg_t in arg_ts) {
        if (arg_t.mode != ty::mo_val) {
            auto arg = args.(i);
            auto root = expr_root(cx, arg, false);
            if (arg_t.mode == ty::mo_alias(true)) {
                alt (path_def_id(cx, arg)) {
                    case (some(?did)) {
                        vec::push(mut_roots, tup(i, did._1));
                    }
                    case (_) {
                        if (!mut_field(root.ds)) {
                            auto m =
                                "passing a temporary value or \
                                 immutable field by mutable alias";
                            cx.tcx.sess.span_fatal(arg.span, m);
                        }
                    }
                }
            }
            alt (path_def_id(cx, root.ex)) {
                case (some(?did)) { vec::push(roots, did._1); }
                case (_) { }
            }
            alt (inner_mut(root.ds)) {
                case (some(?t)) {
                    vec::push(unsafe_ts, t);
                    vec::push(unsafe_t_offsets, i);
                }
                case (_) { }
            }
        }
        i += 1u;
    }
    if (vec::len(unsafe_ts) > 0u) {
        alt (f.node) {
            case (ast::expr_path(_, ?id)) {
                if (def_is_local(cx.tcx.def_map.get(id), true)) {
                    cx.tcx.sess.span_fatal(f.span,
                                         #fmt("function may alias with \
                         argument %u, which is not immutably rooted",
                                              unsafe_t_offsets.(0)));
                }
            }
            case (_) { }
        }
    }
    auto j = 0u;
    for (ty::t unsafe in unsafe_ts) {
        auto offset = unsafe_t_offsets.(j);
        j += 1u;
        auto i = 0u;
        for (ty::arg arg_t in arg_ts) {
            auto mut_alias = arg_t.mode == ty::mo_alias(true);
            if (i != offset &&
                    ty_can_unsafely_include(cx, unsafe, arg_t.ty, mut_alias))
               {
                cx.tcx.sess.span_fatal(args.(i).span,
                                     #fmt("argument %u may alias with \
                     argument %u, which is not immutably rooted",
                                          i, offset));
            }
            i += 1u;
        }
    }
    // Ensure we're not passing a root by mutable alias.

    for (tup(uint, node_id) root in mut_roots) {
        auto mut_alias_to_root = vec::count(root._1, roots) > 1u;
        for (restrict r in sc) {
            if (vec::member(root._1, r.root_vars)) {
                mut_alias_to_root = true;
            }
        }
        if (mut_alias_to_root) {
            cx.tcx.sess.span_fatal(args.(root._0).span,
                                 "passing a mutable alias to a \
                 variable that roots another alias");
        }
    }
    ret rec(root_vars=roots, unsafe_ts=unsafe_ts);
}

fn check_tail_call(&ctx cx, &@ast::expr call) {
    auto args;
    auto f =
        alt (call.node) {
            case (ast::expr_call(?f, ?args_, _)) { args = args_; f }
        };
    auto i = 0u;
    for (ty::arg arg_t in fty_args(cx, ty::expr_ty(*cx.tcx, f))) {
        if (arg_t.mode != ty::mo_val) {
            auto mut_a = arg_t.mode == ty::mo_alias(true);
            auto ok = true;
            alt (args.(i).node) {
                case (ast::expr_path(_, ?id)) {
                    auto def = cx.tcx.def_map.get(id);
                    auto dnum = ast::def_id_of_def(def)._1;
                    alt (cx.local_map.find(dnum)) {
                        case (some(arg(ast::alias(?mut)))) {
                            if (mut_a && !mut) {
                                cx.tcx.sess.span_fatal(args.(i).span,
                                                      "passing an immutable \
                                     alias by mutable alias");
                            }
                        }
                        case (_) { ok = !def_is_local(def, false); }
                    }
                }
                case (_) { ok = false; }
            }
            if (!ok) {
                cx.tcx.sess.span_fatal(args.(i).span,
                                      "can not pass a local value by \
                                     alias to a tail call");
            }
        }
        i += 1u;
    }
}

fn check_alt(&ctx cx, &@ast::expr input, &vec[ast::arm] arms, &scope sc,
             &vt[scope] v) {
    visit::visit_expr(input, sc, v);
    auto root = expr_root(cx, input, true);
    auto roots =
        alt (path_def_id(cx, root.ex)) {
            case (some(?did)) { [did._1] }
            case (_) { [] }
        };
    let vec[ty::t] forbidden_tp =
        alt (inner_mut(root.ds)) { case (some(?t)) { [t] } case (_) { [] } };
    for (ast::arm a in arms) {
        auto dnums = arm_defnums(a);
        auto new_sc = sc;
        if (vec::len(dnums) > 0u) {
            new_sc =
                sc +
                    [@rec(root_vars=roots,
                          block_defnum=dnums.(0),
                          bindings=dnums,
                          tys=forbidden_tp,
                          depends_on=deps(sc, roots),
                          mutable ok=valid)];
        }
        visit::visit_arm(a, new_sc, v);
    }
}

fn arm_defnums(&ast::arm arm) -> vec[node_id] {
    auto dnums = [];
    fn walk_pat(&mutable vec[node_id] found, &@ast::pat p) {
        alt (p.node) {
            case (ast::pat_bind(_, ?id)) { vec::push(found, id); }
            case (ast::pat_tag(_, ?children, _)) {
                for (@ast::pat child in children) { walk_pat(found, child); }
            }
            case (_) { }
        }
    }
    walk_pat(dnums, arm.pat);
    ret dnums;
}

fn check_for_each(&ctx cx, &@ast::local local, &@ast::expr call,
                  &ast::block block, &scope sc, &vt[scope] v) {
    visit::visit_expr(call, sc, v);
    alt (call.node) {
        case (ast::expr_call(?f, ?args, _)) {
            auto data = check_call(cx, f, args, sc);
            auto defnum = local.node.id;
            auto new_sc =
                @rec(root_vars=data.root_vars,
                     block_defnum=defnum,
                     bindings=[defnum],
                     tys=data.unsafe_ts,
                     depends_on=deps(sc, data.root_vars),
                     mutable ok=valid);
            visit::visit_block(block, sc + [new_sc], v);
        }
    }
}

fn check_for(&ctx cx, &@ast::local local, &@ast::expr seq, &ast::block block,
             &scope sc, &vt[scope] v) {
    visit::visit_expr(seq, sc, v);
    auto defnum = local.node.id;
    auto root = expr_root(cx, seq, false);
    auto root_def =
        alt (path_def_id(cx, root.ex)) {
            case (some(?did)) { [did._1] }
            case (_) { [] }
        };
    auto unsafe =
        alt (inner_mut(root.ds)) { case (some(?t)) { [t] } case (_) { [] } };
    // If this is a mutable vector, don't allow it to be touched.

    auto seq_t = ty::expr_ty(*cx.tcx, seq);
    alt (ty::struct(*cx.tcx, seq_t)) {
        case (ty::ty_vec(?mt)) {
            if (mt.mut != ast::imm) { unsafe = [seq_t]; }
        }
        case (ty::ty_str) { /* no-op */ }
        case (ty::ty_ivec(?mt)) {
            if (mt.mut != ast::imm) { unsafe = [seq_t]; }
        }
        case (ty::ty_istr) { /* no-op */ }
    }
    auto new_sc =
        @rec(root_vars=root_def,
             block_defnum=defnum,
             bindings=[defnum],<commit_msg>rustc: Output an unimplemented message when alias encounters an unknown sequence type<commit_after>
import front::ast;
import front::ast::ident;
import front::ast::node_id;
import front::ast::def_id;
import util::common::span;
import visit::vt;
import std::vec;
import std::str;
import std::option;
import std::option::some;
import std::option::none;
import std::option::is_none;


// This is not an alias-analyser (though it would merit from becoming one, or
// at getting input from one, to be more precise). It is a pass that checks
// whether aliases are used in a safe way. Beyond that, though it doesn't have
// a lot to do with aliases, it also checks whether assignments are valid
// (using an lval, which is actually mutable), since it already has all the
// information needed to do that (and the typechecker, which would be a
// logical place for such a check, doesn't).
tag valid { valid; overwritten(span, ast::path); val_taken(span, ast::path); }

type restrict =
    @rec(vec[node_id] root_vars,
         node_id block_defnum,
         vec[node_id] bindings,
         vec[ty::t] tys,
         vec[uint] depends_on,
         mutable valid ok);

type scope = vec[restrict];

tag local_info { arg(ast::mode); objfield(ast::mutability); }

type ctx = rec(@ty::ctxt tcx,
               std::map::hashmap[node_id, local_info] local_map);

fn check_crate(@ty::ctxt tcx, &@ast::crate crate) {
    auto cx =
        @rec(tcx=tcx,

             // Stores information about object fields and function
             // arguments that's otherwise not easily available.
             local_map=util::common::new_int_hash());
    auto v =
        @rec(visit_fn=bind visit_fn(cx, _, _, _, _, _, _, _),
             visit_item=bind visit_item(cx, _, _, _),
             visit_expr=bind visit_expr(cx, _, _, _)
             with *visit::default_visitor[scope]());
    visit::visit_crate(*crate, [], visit::vtor(v));
}

fn visit_fn(@ctx cx, &ast::_fn f, &vec[ast::ty_param] tp, &span sp,
            &ident name, ast::node_id id, &scope sc, &vt[scope] v) {
    visit::visit_fn_decl(f.decl, sc, v);
    for (ast::arg arg_ in f.decl.inputs) {
        cx.local_map.insert(arg_.id, arg(arg_.mode));
    }
    vt(v).visit_block(f.body, [], v);
}

fn visit_item(@ctx cx, &@ast::item i, &scope sc, &vt[scope] v) {
    alt (i.node) {
        case (ast::item_obj(?o, _, _)) {
            for (ast::obj_field f in o.fields) {
                cx.local_map.insert(f.id, objfield(f.mut));
            }
        }
        case (_) { }
    }
    visit::visit_item(i, sc, v);
}

fn visit_expr(@ctx cx, &@ast::expr ex, &scope sc, &vt[scope] v) {
    auto handled = true;
    alt (ex.node) {
        case (ast::expr_call(?f, ?args, _)) {
            check_call(*cx, f, args, sc);
            handled = false;
        }
        case (ast::expr_be(?cl, _)) {
            check_tail_call(*cx, cl);
            visit::visit_expr(cl, sc, v);
        }
        case (ast::expr_alt(?input, ?arms, _)) {
            check_alt(*cx, input, arms, sc, v);
        }
        case (ast::expr_put(?val, _)) {
            alt (val) {
                case (some(?ex)) {
                    auto root = expr_root(*cx, ex, false);
                    if (mut_field(root.ds)) {
                        cx.tcx.sess.span_fatal(ex.span,
                                             "result of put must be" +
                                                 " immutably rooted");
                    }
                    visit_expr(cx, ex, sc, v);
                }
                case (_) { }
            }
        }
        case (ast::expr_for_each(?decl, ?call, ?block, _)) {
            check_for_each(*cx, decl, call, block, sc, v);
        }
        case (ast::expr_for(?decl, ?seq, ?block, _)) {
            check_for(*cx, decl, seq, block, sc, v);
        }
        case (ast::expr_path(?pt, ?id)) {
            check_var(*cx, ex, pt, id, false, sc);
            handled = false;
        }
        case (ast::expr_move(?dest, ?src, _)) {
            check_assign(cx, dest, src, sc, v);
        }
        case (ast::expr_assign(?dest, ?src, _)) {
            check_assign(cx, dest, src, sc, v);
        }
        case (ast::expr_assign_op(_, ?dest, ?src, _)) {
            check_assign(cx, dest, src, sc, v);
        }
        case (_) { handled = false; }
    }
    if (!handled) { visit::visit_expr(ex, sc, v); }
}

fn check_call(&ctx cx, &@ast::expr f, &vec[@ast::expr] args, &scope sc) ->
   rec(vec[node_id] root_vars, vec[ty::t] unsafe_ts) {
    auto fty = ty::expr_ty(*cx.tcx, f);
    auto arg_ts = fty_args(cx, fty);
    let vec[node_id] roots = [];
    let vec[tup(uint, node_id)] mut_roots = [];
    let vec[ty::t] unsafe_ts = [];
    let vec[uint] unsafe_t_offsets = [];
    auto i = 0u;
    for (ty::arg arg_t in arg_ts) {
        if (arg_t.mode != ty::mo_val) {
            auto arg = args.(i);
            auto root = expr_root(cx, arg, false);
            if (arg_t.mode == ty::mo_alias(true)) {
                alt (path_def_id(cx, arg)) {
                    case (some(?did)) {
                        vec::push(mut_roots, tup(i, did._1));
                    }
                    case (_) {
                        if (!mut_field(root.ds)) {
                            auto m =
                                "passing a temporary value or \
                                 immutable field by mutable alias";
                            cx.tcx.sess.span_fatal(arg.span, m);
                        }
                    }
                }
            }
            alt (path_def_id(cx, root.ex)) {
                case (some(?did)) { vec::push(roots, did._1); }
                case (_) { }
            }
            alt (inner_mut(root.ds)) {
                case (some(?t)) {
                    vec::push(unsafe_ts, t);
                    vec::push(unsafe_t_offsets, i);
                }
                case (_) { }
            }
        }
        i += 1u;
    }
    if (vec::len(unsafe_ts) > 0u) {
        alt (f.node) {
            case (ast::expr_path(_, ?id)) {
                if (def_is_local(cx.tcx.def_map.get(id), true)) {
                    cx.tcx.sess.span_fatal(f.span,
                                         #fmt("function may alias with \
                         argument %u, which is not immutably rooted",
                                              unsafe_t_offsets.(0)));
                }
            }
            case (_) { }
        }
    }
    auto j = 0u;
    for (ty::t unsafe in unsafe_ts) {
        auto offset = unsafe_t_offsets.(j);
        j += 1u;
        auto i = 0u;
        for (ty::arg arg_t in arg_ts) {
            auto mut_alias = arg_t.mode == ty::mo_alias(true);
            if (i != offset &&
                    ty_can_unsafely_include(cx, unsafe, arg_t.ty, mut_alias))
               {
                cx.tcx.sess.span_fatal(args.(i).span,
                                     #fmt("argument %u may alias with \
                     argument %u, which is not immutably rooted",
                                          i, offset));
            }
            i += 1u;
        }
    }
    // Ensure we're not passing a root by mutable alias.

    for (tup(uint, node_id) root in mut_roots) {
        auto mut_alias_to_root = vec::count(root._1, roots) > 1u;
        for (restrict r in sc) {
            if (vec::member(root._1, r.root_vars)) {
                mut_alias_to_root = true;
            }
        }
        if (mut_alias_to_root) {
            cx.tcx.sess.span_fatal(args.(root._0).span,
                                 "passing a mutable alias to a \
                 variable that roots another alias");
        }
    }
    ret rec(root_vars=roots, unsafe_ts=unsafe_ts);
}

fn check_tail_call(&ctx cx, &@ast::expr call) {
    auto args;
    auto f =
        alt (call.node) {
            case (ast::expr_call(?f, ?args_, _)) { args = args_; f }
        };
    auto i = 0u;
    for (ty::arg arg_t in fty_args(cx, ty::expr_ty(*cx.tcx, f))) {
        if (arg_t.mode != ty::mo_val) {
            auto mut_a = arg_t.mode == ty::mo_alias(true);
            auto ok = true;
            alt (args.(i).node) {
                case (ast::expr_path(_, ?id)) {
                    auto def = cx.tcx.def_map.get(id);
                    auto dnum = ast::def_id_of_def(def)._1;
                    alt (cx.local_map.find(dnum)) {
                        case (some(arg(ast::alias(?mut)))) {
                            if (mut_a && !mut) {
                                cx.tcx.sess.span_fatal(args.(i).span,
                                                      "passing an immutable \
                                     alias by mutable alias");
                            }
                        }
                        case (_) { ok = !def_is_local(def, false); }
                    }
                }
                case (_) { ok = false; }
            }
            if (!ok) {
                cx.tcx.sess.span_fatal(args.(i).span,
                                      "can not pass a local value by \
                                     alias to a tail call");
            }
        }
        i += 1u;
    }
}

fn check_alt(&ctx cx, &@ast::expr input, &vec[ast::arm] arms, &scope sc,
             &vt[scope] v) {
    visit::visit_expr(input, sc, v);
    auto root = expr_root(cx, input, true);
    auto roots =
        alt (path_def_id(cx, root.ex)) {
            case (some(?did)) { [did._1] }
            case (_) { [] }
        };
    let vec[ty::t] forbidden_tp =
        alt (inner_mut(root.ds)) { case (some(?t)) { [t] } case (_) { [] } };
    for (ast::arm a in arms) {
        auto dnums = arm_defnums(a);
        auto new_sc = sc;
        if (vec::len(dnums) > 0u) {
            new_sc =
                sc +
                    [@rec(root_vars=roots,
                          block_defnum=dnums.(0),
                          bindings=dnums,
                          tys=forbidden_tp,
                          depends_on=deps(sc, roots),
                          mutable ok=valid)];
        }
        visit::visit_arm(a, new_sc, v);
    }
}

fn arm_defnums(&ast::arm arm) -> vec[node_id] {
    auto dnums = [];
    fn walk_pat(&mutable vec[node_id] found, &@ast::pat p) {
        alt (p.node) {
            case (ast::pat_bind(_, ?id)) { vec::push(found, id); }
            case (ast::pat_tag(_, ?children, _)) {
                for (@ast::pat child in children) { walk_pat(found, child); }
            }
            case (_) { }
        }
    }
    walk_pat(dnums, arm.pat);
    ret dnums;
}

fn check_for_each(&ctx cx, &@ast::local local, &@ast::expr call,
                  &ast::block block, &scope sc, &vt[scope] v) {
    visit::visit_expr(call, sc, v);
    alt (call.node) {
        case (ast::expr_call(?f, ?args, _)) {
            auto data = check_call(cx, f, args, sc);
            auto defnum = local.node.id;
            auto new_sc =
                @rec(root_vars=data.root_vars,
                     block_defnum=defnum,
                     bindings=[defnum],
                     tys=data.unsafe_ts,
                     depends_on=deps(sc, data.root_vars),
                     mutable ok=valid);
            visit::visit_block(block, sc + [new_sc], v);
        }
    }
}

fn check_for(&ctx cx, &@ast::local local, &@ast::expr seq, &ast::block block,
             &scope sc, &vt[scope] v) {
    visit::visit_expr(seq, sc, v);
    auto defnum = local.node.id;
    auto root = expr_root(cx, seq, false);
    auto root_def =
        alt (path_def_id(cx, root.ex)) {
            case (some(?did)) { [did._1] }
            case (_) { [] }
        };
    auto unsafe =
        alt (inner_mut(root.ds)) { case (some(?t)) { [t] } case (_) { [] } };
    // If this is a mutable vector, don't allow it to be touched.

    auto seq_t = ty::expr_ty(*cx.tcx, seq);
    alt (ty::struct(*cx.tcx, seq_t)) {
        case (ty::ty_vec(?mt)) {
            if (mt.mut != ast::imm) { unsafe = [seq_t]; }
        }
        case (ty::ty_str) { /* no-op */ }
        case (ty::ty_ivec(?mt)) {
            if (mt.mut != ast::imm) { unsafe = [seq_t]; }
        }
        case (ty::ty_istr) { /* no-op */ }
        case (_) {
            cx.tcx.sess.span_unimpl(seq.span, "unknown seq type " +
                                    pretty::ppaux::ty_to_str(*cx.tcx, seq_t));
        }
    }
    auto new_sc =
        @rec(root_vars=root_def,
             block_defnum=defnum,
             bindings=[defnum],
{'fraction_non_alphanumeric': 0.14304812834224598, 'fraction_numerical': 0.001701507049100632, 'mean_word_length': 1.472208312468703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 39, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6926307', 'n_tokens_mistral': 8075, 'n_tokens_neox': 7875, 'n_words': 1813}