<commit_before>import std::vec;
import std::uint;
import std::str;
import std::term;
import std::io;
import std::option;
import std::option::some;
import std::option::none;

type filename = str;

type file_pos = {ch: uint, byte: uint};

/* A codemap is a thing that maps uints to file/line/column positions
 * in a crate. This to make it possible to represent the positions
 * with single-word things, rather than passing records all over the
 * compiler.
 */
type filemap =
    @{name: filename, start_pos: file_pos, mutable lines: [file_pos]};

type codemap = @{mutable files: [filemap]};

type loc = {filename: filename, line: uint, col: uint};

fn new_codemap() -> codemap { ret @{mutable files: []}; }

fn new_filemap(filename: filename, start_pos_ch: uint, start_pos_byte: uint)
   -> filemap {
    ret @{name: filename,
          start_pos: {ch: start_pos_ch, byte: start_pos_byte},
          mutable lines: [{ch: start_pos_ch, byte: start_pos_byte}]};
}

fn next_line(file: filemap, chpos: uint, byte_pos: uint) {
    file.lines += [{ch: chpos, byte: byte_pos}];
}

type lookup_fn = fn(file_pos) -> uint;

fn lookup_pos(map: codemap, pos: uint, lookup: lookup_fn) -> loc {
    let a = 0u;
    let b = vec::len(map.files);
    while b - a > 1u {
        let m = (a + b) / 2u;
        if lookup(map.files[m].start_pos) > pos { b = m; } else { a = m; }
    }
    let f = map.files[a];
    a = 0u;
    b = vec::len(f.lines);
    while b - a > 1u {
        let m = (a + b) / 2u;
        if lookup(f.lines[m]) > pos { b = m; } else { a = m; }
    }
    ret {filename: f.name, line: a + 1u, col: pos - lookup(f.lines[a])};
}

fn lookup_char_pos(map: codemap, pos: uint) -> loc {
    fn lookup(pos: file_pos) -> uint { ret pos.ch; }
    ret lookup_pos(map, pos, lookup);
}

fn lookup_byte_pos(map: codemap, pos: uint) -> loc {
    fn lookup(pos: file_pos) -> uint { ret pos.byte; }
    ret lookup_pos(map, pos, lookup);
}

tag opt_span {
     //hack (as opposed to option::t), to make `span` compile
    os_none;
    os_some(@span);
}
type span = {lo: uint, hi: uint, expanded_from: opt_span};

fn span_to_str(sp: &span, cm: &codemap) -> str {
    let cur = sp;
    let res = "";
    let prev_file = none;
    while true {
        let lo = lookup_char_pos(cm, cur.lo);
        let hi = lookup_char_pos(cm, cur.hi);
        res +=
            #fmt["%s:%u:%u:%u:%u",
                 if some(lo.filename) == prev_file {
                     "-"
                 } else { lo.filename }, lo.line, lo.col, hi.line, hi.col];
        alt cur.expanded_from {
          os_none. { break; }
          os_some(new_sp) {
            cur = *new_sp;
            prev_file = some(lo.filename);
            res += "<<";
          }
        }
    }

    ret res;
}

fn emit_diagnostic(sp: &option::t<span>, msg: &str, kind: &str, color: u8,
                   cm: &codemap) {
    let ss = "<input>:0:0:0:0";
    let maybe_lines: option::t<@file_lines> = none;
    alt sp {
      some(ssp) {
        ss = span_to_str(ssp, cm);
        maybe_lines = some(span_to_lines(ssp, cm));
      }
      none. { }
    }
    io::stdout().write_str(ss + ": ");
    if term::color_supported() {
        term::fg(io::stdout().get_buf_writer(), color);
    }
    io::stdout().write_str(#fmt["%s:", kind]);
    if term::color_supported() { term::reset(io::stdout().get_buf_writer()); }
    io::stdout().write_str(#fmt[" %s\n", msg]);

    maybe_highlight_lines(sp, cm, maybe_lines);
}

fn maybe_highlight_lines(sp: &option::t<span>, cm: &codemap,
                         maybe_lines: option::t<@file_lines>) {

    alt maybe_lines {
      some(lines) {
        // If we're not looking at a real file then we can't re-open it to
        // pull out the lines
        if lines.name == "-" { ret; }

        // FIXME: reading in the entire file is the worst possible way to
        //        get access to the necessary lines.
        let file = io::read_whole_file_str(lines.name);
        let fm = get_filemap(cm, lines.name);

        // arbitrarily only print up to six lines of the error
        let max_lines = 6u;
        let elided = false;
        let display_lines = lines.lines;
        if vec::len(display_lines) > max_lines {<commit_msg>Output error spans that don't confuse emacs compilation mode. Closes #845<commit_after>import std::vec;
import std::uint;
import std::str;
import std::term;
import std::io;
import std::option;
import std::option::some;
import std::option::none;

type filename = str;

type file_pos = {ch: uint, byte: uint};

/* A codemap is a thing that maps uints to file/line/column positions
 * in a crate. This to make it possible to represent the positions
 * with single-word things, rather than passing records all over the
 * compiler.
 */
type filemap =
    @{name: filename, start_pos: file_pos, mutable lines: [file_pos]};

type codemap = @{mutable files: [filemap]};

type loc = {filename: filename, line: uint, col: uint};

fn new_codemap() -> codemap { ret @{mutable files: []}; }

fn new_filemap(filename: filename, start_pos_ch: uint, start_pos_byte: uint)
   -> filemap {
    ret @{name: filename,
          start_pos: {ch: start_pos_ch, byte: start_pos_byte},
          mutable lines: [{ch: start_pos_ch, byte: start_pos_byte}]};
}

fn next_line(file: filemap, chpos: uint, byte_pos: uint) {
    file.lines += [{ch: chpos, byte: byte_pos}];
}

type lookup_fn = fn(file_pos) -> uint;

fn lookup_pos(map: codemap, pos: uint, lookup: lookup_fn) -> loc {
    let a = 0u;
    let b = vec::len(map.files);
    while b - a > 1u {
        let m = (a + b) / 2u;
        if lookup(map.files[m].start_pos) > pos { b = m; } else { a = m; }
    }
    let f = map.files[a];
    a = 0u;
    b = vec::len(f.lines);
    while b - a > 1u {
        let m = (a + b) / 2u;
        if lookup(f.lines[m]) > pos { b = m; } else { a = m; }
    }
    ret {filename: f.name, line: a + 1u, col: pos - lookup(f.lines[a])};
}

fn lookup_char_pos(map: codemap, pos: uint) -> loc {
    fn lookup(pos: file_pos) -> uint { ret pos.ch; }
    ret lookup_pos(map, pos, lookup);
}

fn lookup_byte_pos(map: codemap, pos: uint) -> loc {
    fn lookup(pos: file_pos) -> uint { ret pos.byte; }
    ret lookup_pos(map, pos, lookup);
}

tag opt_span {
     //hack (as opposed to option::t), to make `span` compile
    os_none;
    os_some(@span);
}
type span = {lo: uint, hi: uint, expanded_from: opt_span};

fn span_to_str(sp: &span, cm: &codemap) -> str {
    let cur = sp;
    let res = "";
    let prev_file = none;
    while true {
        let lo = lookup_char_pos(cm, cur.lo);
        let hi = lookup_char_pos(cm, cur.hi);
        res +=
            #fmt["%s:%u:%u: %u:%u",
                 if some(lo.filename) == prev_file {
                     "-"
                 } else { lo.filename }, lo.line, lo.col, hi.line, hi.col];
        alt cur.expanded_from {
          os_none. { break; }
          os_some(new_sp) {
            cur = *new_sp;
            prev_file = some(lo.filename);
            res += "<<";
          }
        }
    }

    ret res;
}

fn emit_diagnostic(sp: &option::t<span>, msg: &str, kind: &str, color: u8,
                   cm: &codemap) {
    let ss = "<input>:0:0:0:0";
    let maybe_lines: option::t<@file_lines> = none;
    alt sp {
      some(ssp) {
        ss = span_to_str(ssp, cm);
        maybe_lines = some(span_to_lines(ssp, cm));
      }
      none. { }
    }
    io::stdout().write_str(ss + " ");
    if term::color_supported() {
        term::fg(io::stdout().get_buf_writer(), color);
    }
    io::stdout().write_str(#fmt["%s:", kind]);
    if term::color_supported() { term::reset(io::stdout().get_buf_writer()); }
    io::stdout().write_str(#fmt[" %s\n", msg]);

    maybe_highlight_lines(sp, cm, maybe_lines);
}

fn maybe_highlight_lines(sp: &option::t<span>, cm: &codemap,
                         maybe_lines: option::t<@file_lines>) {

    alt maybe_lines {
      some(lines) {
        // If we're not looking at a real file then we can't re-open it to
        // pull out the lines
        if lines.name == "-" { ret; }

        // FIXME: reading in the entire file is the worst possible way to
        //        get access to the necessary lines.
        let file = io::read_whole_file_str(lines.name);
        let fm = get_filemap(cm, lines.name);

        // arbitrarily only print up to six lines of the error
        let max_lines = 6u;
        let elided = false;
        let display_lines = lines.lines;
        if vec::len(display_lines) > max_lines {
{'fraction_non_alphanumeric': 0.15130803676643884, 'fraction_numerical': 0.0034173933537591326, 'mean_word_length': 2.4542124542124544, 'pattern_counts': {'":': 1, '<': 17, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7218433', 'n_tokens_mistral': 3024, 'n_tokens_neox': 2975, 'n_words': 833}