<commit_before>#![feature(plugin_registrar, rustc_private, convert, slice_patterns)]

extern crate rustc;
extern crate syntax;

use std::rc::Rc;

use syntax::codemap::Span;
use syntax::parse::{PResult, token};
use syntax::ast::{Delimited, Ident, TokenTree, TtDelimited, TtToken};
use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager};
use syntax::ext::quote::rt::ToTokens;
use syntax::util::small_vector::SmallVector;
use rustc::plugin::Registry;


fn tt_vec(tts: &[TokenTree]) -> Vec<TokenTree> {
    tts.iter().map(|tt| tt.clone()).collect()
}

fn ident_of_ctx_and_span(cx: &mut ExtCtxt, span: Span, ident: &str) -> TokenTree {
    TtToken(span, token::Ident(cx.ident_of(ident), token::Plain))
}

fn build_string_from_idents(target: &mut String, tts: &[TokenTree]) -> Result<(), Span> {
    for t in tts {
        match t {
            &TtToken(_, token::Ident(ref i, _)) => {
                target.extend(i.as_str().chars());
            },
            &TtToken(_, token::Comma) => (),
            &TtDelimited(delim_span, ref delim) => {
                if delim.delim != token::DelimToken::Bracket {
                    return Err(delim_span);
                }
                try!(build_string_from_idents(target, &delim.tts[..]));
            }
            ref tt => return Err(tt.get_span()),
        }
    }
    Ok(())
}

fn fnconcat(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree])
            -> Box<MacResult + 'static> {
    if args.is_empty() {
        cx.span_err(sp, "fnconcat! can't be called with nothing");
        return DummyResult::any(sp);
    }
    let mut fn_tokens = tt_vec(args);
    fn_tokens[0] = if let &TtDelimited(delim_span, ref delim) = &fn_tokens[0] {
        if delim.delim != token::DelimToken::Bracket {
            cx.span_err(delim_span, "use square brackets (i.e. [..]) in place of the function name");
            return DummyResult::any(delim_span);
        }
        let mut concatenated = String::new();
        match build_string_from_idents(&mut concatenated, &delim.tts) {
            Ok(()) => (),
            Err(err_span) => {
                cx.span_err(err_span, "non-ident, non-comma token");
                return DummyResult::any(sp);
            },
        }
        if concatenated.is_empty() {
            cx.span_err(delim_span, "empty identifier");
            return DummyResult::any(sp);
        }
        ident_of_ctx_and_span(cx, delim_span, concatenated.as_str())
    } else {
        cx.span_err(sp, "use square brackets (i.e. [..]) in place of the function name");
        return DummyResult::any(sp);
    };
    fn_tokens.insert(0, ident_of_ctx_and_span(cx, sp, "fn"));
    let mut p = cx.new_parser_from_tts(&fn_tokens[..]);
    if let Some(i) = p.parse_item() {
        return MacEager::items(SmallVector::one(i));
    }
    cx.span_err(sp, "couldn't parse a fn");
    return DummyResult::any(sp);
}


fn pull_tts_from_paren_groups(tts: &[TokenTree]) -> PResult<Vec<Vec<TokenTree>>> {
    let mut ret = Vec::new();
    for t in tts {
        match t {
            &TtDelimited(_, ref delim) if delim.delim == token::DelimToken::Paren => {
                ret.push(tt_vec(&delim.tts[..]));
            },
            _ => {

            },
        }
    }<commit_msg>Correct some incorrect spans.<commit_after>#![feature(plugin_registrar, rustc_private, convert, slice_patterns)]

extern crate rustc;
extern crate syntax;

use std::rc::Rc;

use syntax::codemap::Span;
use syntax::parse::{PResult, token};
use syntax::ast::{Delimited, Ident, TokenTree, TtDelimited, TtToken};
use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager};
use syntax::ext::quote::rt::ToTokens;
use syntax::util::small_vector::SmallVector;
use rustc::plugin::Registry;


fn tt_vec(tts: &[TokenTree]) -> Vec<TokenTree> {
    tts.iter().map(|tt| tt.clone()).collect()
}

fn ident_of_ctx_and_span(cx: &mut ExtCtxt, span: Span, ident: &str) -> TokenTree {
    TtToken(span, token::Ident(cx.ident_of(ident), token::Plain))
}

fn build_string_from_idents(target: &mut String, tts: &[TokenTree]) -> Result<(), Span> {
    for t in tts {
        match t {
            &TtToken(_, token::Ident(ref i, _)) => {
                target.extend(i.as_str().chars());
            },
            &TtToken(_, token::Comma) => (),
            &TtDelimited(delim_span, ref delim) => {
                if delim.delim != token::DelimToken::Bracket {
                    return Err(delim_span);
                }
                try!(build_string_from_idents(target, &delim.tts[..]));
            }
            ref tt => return Err(tt.get_span()),
        }
    }
    Ok(())
}

fn fnconcat(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree])
            -> Box<MacResult + 'static> {
    if args.is_empty() {
        cx.span_err(sp, "fnconcat! can't be called with nothing");
        return DummyResult::any(sp);
    }
    let mut fn_tokens = tt_vec(args);
    fn_tokens[0] = if let &TtDelimited(delim_span, ref delim) = &fn_tokens[0] {
        if delim.delim != token::DelimToken::Bracket {
            cx.span_err(delim_span, "use square brackets (i.e. [..]) in place of the function name");
            return DummyResult::any(delim_span);
        }
        let mut concatenated = String::new();
        match build_string_from_idents(&mut concatenated, &delim.tts) {
            Ok(()) => (),
            Err(err_span) => {
                cx.span_err(err_span, "non-ident, non-comma token");
                return DummyResult::any(err_span);
            },
        }
        if concatenated.is_empty() {
            cx.span_err(delim_span, "empty identifier");
            return DummyResult::any(delim_span);
        }
        ident_of_ctx_and_span(cx, delim_span, concatenated.as_str())
    } else {
        cx.span_err(sp, "use square brackets (i.e. [..]) in place of the function name");
        return DummyResult::any(sp);
    };
    fn_tokens.insert(0, ident_of_ctx_and_span(cx, sp, "fn"));
    let mut p = cx.new_parser_from_tts(&fn_tokens[..]);
    if let Some(i) = p.parse_item() {
        return MacEager::items(SmallVector::one(i));
    }
    cx.span_err(sp, "couldn't parse a fn");
    return DummyResult::any(sp);
}


fn pull_tts_from_paren_groups(tts: &[TokenTree]) -> PResult<Vec<Vec<TokenTree>>> {
    let mut ret = Vec::new();
    for t in tts {
        match t {
            &TtDelimited(_, ref delim) if delim.delim == token::DelimToken::Paren => {
                ret.push(tt_vec(&delim.tts[..]));
            },
            _ => {

            },
        }
    }
{'fraction_non_alphanumeric': 0.15779002142638507, 'fraction_numerical': 0.0009182736455463728, 'mean_word_length': 2.5690879300928455, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1110035', 'n_tokens_mistral': 2269, 'n_tokens_neox': 2192, 'n_words': 437}