<commit_before>extern crate std;
extern crate collections;
use std::io::File;
use std::io::BufferedReader;

fn getline(fname : &str, linenum : uint) -> ~str {
    let path = Path::new(fname);
    let mut i = 0;
    let mut file = BufferedReader::new(File::open(&path));
    for line in file.lines() {
        //print!("{}", line);
        i += 1;
        if i == linenum {
            return line.to_owned();
        }
    }
    return ~"not found";
}

fn is_path_char(c : char) -> bool {
    c.is_alphanumeric() || (c == '_') || (c == '!') || (c == ':')
}

fn is_ident_char(c : char) -> bool {
    c.is_alphanumeric() || (c == '_') || (c == '!')
}

fn expand_ident(s : &str, pos : uint) -> (uint,uint) {
    let sb = s.slice_to(pos);
    let mut start = pos;

    // backtrack to find start of word
    for (i, c) in sb.char_indices_rev() {
        if !is_ident_char(c) {
            break;
        }
        start = i;
    }
    return (start, pos);
}

fn expand_searchstr(s : &str, pos : uint) -> ~str {
    let sb = s.slice_to(pos);
    let mut start = pos;

    // backtrack to find start of word
    for (i, c) in sb.char_indices_rev() {
        if !is_path_char(c) {
            break;
        }
        start = i;
    }
    return s.slice(start,pos).to_owned();
}

fn find_end(s : &str, pos : uint) -> uint {
    // find end of word
    let sa = s.slice_from(pos);
    let mut end = pos;
    for (i, c) in sa.char_indices() {
        if !is_path_char(c) {
            break;
        }
        end = pos + i + 1;
    }
    return end;
}


fn find_in_module(path : &Path, s : &str, matchfn : &|&str,uint,&Path,&str|) {
    let mut file = BufferedReader::new(File::open(path));
    //let modsearchstr = "pub mod "+s;
    let modsearchstr = "mod ";
    let fnsearchstr = "fn ";
    let structsearchstr = "struct ";
    let cratesearchstr = "extern crate ";
    let mut i = 0;
    for line in file.lines() {
        i += 1;
        match line.find_str(modsearchstr+s) {
            Some(n) => {
                let end = find_end(line, n+modsearchstr.len());
                let l = line.slice(n + modsearchstr.len(), end);
                (*matchfn)(l, i, path, line);
            }
            None => {}
        }
        match line.find_str(fnsearchstr+s) {
            Some(n) => {
                let end = find_end(line, n+3);
                let l = line.slice(n + 3, end);
                (*matchfn)(l + "(", i, path, line);
            }
            None => {}
        }
        match line.find_str(structsearchstr+s) {
            Some(n) => {
                let end = find_end(line, n+7);
                let l = line.slice(n+7, end);
                (*matchfn)(l, i, path, line);
            }
            None => {}
        }

        match line.find_str(cratesearchstr+s) {
            Some(n) => {
                let end = find_end(line, n+ cratesearchstr.len());
                let cratename = line.slice(n + cratesearchstr.len(), end);
                (*matchfn)(cratename+"::", i, path, line)
            }
            None => {}
        }<commit_msg>Stop putting braces after fn name completes - needs to handle this based on context so disable for now<commit_after>extern crate std;
extern crate collections;
use std::io::File;
use std::io::BufferedReader;

fn getline(fname : &str, linenum : uint) -> ~str {
    let path = Path::new(fname);
    let mut i = 0;
    let mut file = BufferedReader::new(File::open(&path));
    for line in file.lines() {
        //print!("{}", line);
        i += 1;
        if i == linenum {
            return line.to_owned();
        }
    }
    return ~"not found";
}

fn is_path_char(c : char) -> bool {
    c.is_alphanumeric() || (c == '_') || (c == '!') || (c == ':')
}

fn is_ident_char(c : char) -> bool {
    c.is_alphanumeric() || (c == '_') || (c == '!')
}

fn expand_ident(s : &str, pos : uint) -> (uint,uint) {
    let sb = s.slice_to(pos);
    let mut start = pos;

    // backtrack to find start of word
    for (i, c) in sb.char_indices_rev() {
        if !is_ident_char(c) {
            break;
        }
        start = i;
    }
    return (start, pos);
}

fn expand_searchstr(s : &str, pos : uint) -> ~str {
    let sb = s.slice_to(pos);
    let mut start = pos;

    // backtrack to find start of word
    for (i, c) in sb.char_indices_rev() {
        if !is_path_char(c) {
            break;
        }
        start = i;
    }
    return s.slice(start,pos).to_owned();
}

fn find_end(s : &str, pos : uint) -> uint {
    // find end of word
    let sa = s.slice_from(pos);
    let mut end = pos;
    for (i, c) in sa.char_indices() {
        if !is_path_char(c) {
            break;
        }
        end = pos + i + 1;
    }
    return end;
}


fn find_in_module(path : &Path, s : &str, matchfn : &|&str,uint,&Path,&str|) {
    let mut file = BufferedReader::new(File::open(path));
    //let modsearchstr = "pub mod "+s;
    let modsearchstr = "mod ";
    let fnsearchstr = "fn ";
    let structsearchstr = "struct ";
    let cratesearchstr = "extern crate ";
    let mut i = 0;
    for line in file.lines() {
        i += 1;
        match line.find_str(modsearchstr+s) {
            Some(n) => {
                let end = find_end(line, n+modsearchstr.len());
                let l = line.slice(n + modsearchstr.len(), end);
                (*matchfn)(l, i, path, line);
            }
            None => {}
        }
        match line.find_str(fnsearchstr+s) {
            Some(n) => {
                let end = find_end(line, n+3);
                let l = line.slice(n + 3, end);
                (*matchfn)(l, i, path, line);
            }
            None => {}
        }
        match line.find_str(structsearchstr+s) {
            Some(n) => {
                let end = find_end(line, n+7);
                let l = line.slice(n+7, end);
                (*matchfn)(l, i, path, line);
            }
            None => {}
        }

        match line.find_str(cratesearchstr+s) {
            Some(n) => {
                let end = find_end(line, n+ cratesearchstr.len());
                let cratename = line.slice(n + cratesearchstr.len(), end);
                (*matchfn)(cratename+"::", i, path, line)
            }
            None => {}
        }
{'fraction_non_alphanumeric': 0.15862068965517243, 'fraction_numerical': 0.0028869286287089013, 'mean_word_length': 1.7089487402258905, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2428183', 'n_tokens_mistral': 2167, 'n_tokens_neox': 2114, 'n_words': 535}