<commit_before>use std::cmp;
use std::fmt::{mod, Show, Formatter};
use std::io::fs;
use glob::Pattern;
use git2;

use core::{Package, PackageId, Summary, SourceId, Source, Dependency, Registry};
use ops;
use util::{CargoResult, internal, internal_error};

pub struct PathSource {
    id: SourceId,
    path: Path,
    updated: bool,
    packages: Vec<Package>
}

// TODO: Figure out if packages should be discovered in new or self should be
// mut and packages are discovered in update
impl PathSource {

    pub fn for_path(path: &Path) -> CargoResult<PathSource> {
        log!(5, "PathSource::for_path; path={}", path.display());
        Ok(PathSource::new(path, &try!(SourceId::for_path(path))))
    }

    /// Invoked with an absolute path to a directory that contains a Cargo.toml.
    /// The source will read the manifest and find any other packages contained
    /// in the directory structure reachable by the root manifest.
    pub fn new(path: &Path, id: &SourceId) -> PathSource {
        log!(5, "new; id={}", id);

        PathSource {
            id: id.clone(),
            path: path.clone(),
            updated: false,
            packages: Vec::new()
        }
    }

    pub fn get_root_package(&self) -> CargoResult<Package> {
        log!(5, "get_root_package; source={}", self);

        if !self.updated {
            return Err(internal("source has not been updated"))
        }

        match self.packages.as_slice().head() {
            Some(pkg) => Ok(pkg.clone()),
            None => Err(internal("no package found in source"))
        }
    }

    fn read_packages(&self) -> CargoResult<Vec<Package>> {
        if self.updated {
            Ok(self.packages.clone())
        } else {
            ops::read_packages(&self.path, &self.id)
        }
    }

    /// List all files relevant to building this package inside this source.
    ///
    /// This function will use the appropriate methods to determine what is the
    /// set of files underneath this source's directory which are relevant for
    /// building `pkg`.
    ///
    /// The basic assumption of this method is that all files in the directory
    /// are relevant for building this package, but it also contains logic to
    /// use other methods like .gitignore to filter the list of files.
    pub fn list_files(&self, pkg: &Package) -> CargoResult<Vec<Path>> {
        let candidates = match git2::Repository::open(&self.path) {
            Ok(repo) => try!(self.list_files_git(pkg, repo)),
            Err(..) => try!(self.list_files_walk(pkg)),
        };

        let pats = pkg.get_manifest().get_exclude().iter().map(|p| {
            Pattern::new(p.as_slice())
        }).collect::<Vec<Pattern>>();

        let root = pkg.get_manifest_path().dir_path();
        Ok(candidates.move_iter().filter(|candidate| {
            let candidate = candidate.path_relative_from(&root).unwrap();
            !pats.iter().any(|p| p.matches_path(&candidate)) &&
                candidate.is_file()
        }).collect())
    }

    fn list_files_git(&self, pkg: &Package, repo: git2::Repository)
                      -> CargoResult<Vec<Path>> {
        let index = try!(repo.index());
        let mut ret = Vec::new();
        'outer: for i in range(0, index.len()) {
            let entry = match index.get(i) { Some(e) => e, None => continue };
            let fname = entry.path.as_bytes_no_nul();
            let path = pkg.get_manifest_path().dir_path().join(fname);

            // Filter out Cargo.lock and target always
            if fname == b"Cargo.lock" { continue }
            if fname == b"target" { continue }

            // Filter out all other packages
            for pkg in self.packages.iter().filter(|p| *p != pkg) {
                let pkg_path = pkg.get_manifest_path().dir_path();
                if pkg_path.is_ancestor_of(&path) { continue 'outer; }
            }

            // We found a file!
            ret.push(path);<commit_msg>Fix bug in list_file_walk when cwd != path<commit_after>use std::cmp;
use std::fmt::{mod, Show, Formatter};
use std::io::fs;
use glob::Pattern;
use git2;

use core::{Package, PackageId, Summary, SourceId, Source, Dependency, Registry};
use ops;
use util::{CargoResult, internal, internal_error};

pub struct PathSource {
    id: SourceId,
    path: Path,
    updated: bool,
    packages: Vec<Package>
}

// TODO: Figure out if packages should be discovered in new or self should be
// mut and packages are discovered in update
impl PathSource {

    pub fn for_path(path: &Path) -> CargoResult<PathSource> {
        log!(5, "PathSource::for_path; path={}", path.display());
        Ok(PathSource::new(path, &try!(SourceId::for_path(path))))
    }

    /// Invoked with an absolute path to a directory that contains a Cargo.toml.
    /// The source will read the manifest and find any other packages contained
    /// in the directory structure reachable by the root manifest.
    pub fn new(path: &Path, id: &SourceId) -> PathSource {
        log!(5, "new; id={}", id);

        PathSource {
            id: id.clone(),
            path: path.clone(),
            updated: false,
            packages: Vec::new()
        }
    }

    pub fn get_root_package(&self) -> CargoResult<Package> {
        log!(5, "get_root_package; source={}", self);

        if !self.updated {
            return Err(internal("source has not been updated"))
        }

        match self.packages.as_slice().head() {
            Some(pkg) => Ok(pkg.clone()),
            None => Err(internal("no package found in source"))
        }
    }

    fn read_packages(&self) -> CargoResult<Vec<Package>> {
        if self.updated {
            Ok(self.packages.clone())
        } else {
            ops::read_packages(&self.path, &self.id)
        }
    }

    /// List all files relevant to building this package inside this source.
    ///
    /// This function will use the appropriate methods to determine what is the
    /// set of files underneath this source's directory which are relevant for
    /// building `pkg`.
    ///
    /// The basic assumption of this method is that all files in the directory
    /// are relevant for building this package, but it also contains logic to
    /// use other methods like .gitignore to filter the list of files.
    pub fn list_files(&self, pkg: &Package) -> CargoResult<Vec<Path>> {
        let candidates = match git2::Repository::open(&self.path) {
            Ok(repo) => try!(self.list_files_git(pkg, repo)),
            Err(..) => try!(self.list_files_walk(pkg)),
        };

        let pats = pkg.get_manifest().get_exclude().iter().map(|p| {
            Pattern::new(p.as_slice())
        }).collect::<Vec<Pattern>>();

        let root = pkg.get_manifest_path().dir_path();
        Ok(candidates.move_iter().filter(|candidate| {
            let relative_path = candidate.path_relative_from(&root).unwrap();
            !pats.iter().any(|p| p.matches_path(&relative_path)) &&
                candidate.is_file()
        }).collect())
    }

    fn list_files_git(&self, pkg: &Package, repo: git2::Repository)
                      -> CargoResult<Vec<Path>> {
        let index = try!(repo.index());
        let mut ret = Vec::new();
        'outer: for i in range(0, index.len()) {
            let entry = match index.get(i) { Some(e) => e, None => continue };
            let fname = entry.path.as_bytes_no_nul();
            let path = pkg.get_manifest_path().dir_path().join(fname);

            // Filter out Cargo.lock and target always
            if fname == b"Cargo.lock" { continue }
            if fname == b"target" { continue }

            // Filter out all other packages
            for pkg in self.packages.iter().filter(|p| *p != pkg) {
                let pkg_path = pkg.get_manifest_path().dir_path();
                if pkg_path.is_ancestor_of(&path) { continue 'outer; }
            }

            // We found a file!
            ret.push(path);
{'fraction_non_alphanumeric': 0.13626649905719673, 'fraction_numerical': 0.0017598994343180389, 'mean_word_length': 2.293046357615894, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4505891', 'n_tokens_mistral': 2324, 'n_tokens_neox': 2261, 'n_words': 671}