<commit_before>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 <NAME> <<EMAIL>> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use std::collections::HashMap;
use std::ops::Drop;
use std::path::PathBuf;
use std::result::Result as RResult;
use std::sync::Arc;
use std::sync::RwLock;
use std::io::Read;
use std::convert::Into;
use std::ops::Deref;
use std::ops::DerefMut;
use std::fmt::Formatter;
use std::fmt::Debug;
use std::fmt::Error as FMTError;

use toml::Value;
use glob::glob;
use walkdir::WalkDir;
use walkdir::Iter as WalkDirIter;

use error::{StoreError as SE, StoreErrorKind as SEK};
use error::MapErrInto;
use storeid::{IntoStoreId, StoreId, StoreIdIterator};
use file_abstraction::FileAbstractionInstance;
use toml_ext::*;
// We re-export the following things so tests can use them
pub use file_abstraction::FileAbstraction;
pub use file_abstraction::FSFileAbstraction;
pub use file_abstraction::InMemoryFileAbstraction;

use libimagerror::into::IntoError;
use libimagerror::trace::trace_error;
use libimagutil::debug_result::*;

use self::glob_store_iter::*;

/// The Result Type returned by any interaction with the store that could fail
pub type Result<T> = RResult<T, SE>;


#[derive(Debug, PartialEq)]
enum StoreEntryStatus {
    Present,
    Borrowed
}

/// A store entry, depending on the option type it is either borrowed currently
/// or not.
#[derive(Debug)]
struct StoreEntry {
    id: StoreId,
    file: Box<FileAbstractionInstance>,
    status: StoreEntryStatus,
}

pub enum StoreObject {
    Id(StoreId),
    Collection(PathBuf),
}

pub struct Walk {
    store_path: PathBuf,
    dirwalker: WalkDirIter,
}

impl Walk {

    fn new(mut store_path: PathBuf, mod_name: &str) -> Walk {
        let pb = store_path.clone();
        store_path.push(mod_name);
        Walk {
            store_path: pb,
            dirwalker: WalkDir::new(store_path).into_iter(),
        }
    }
}

impl ::std::ops::Deref for Walk {
    type Target = WalkDirIter;

    fn deref(&self) -> &Self::Target {
        &self.dirwalker
    }
}

impl Iterator for Walk {
    type Item = StoreObject;

    fn next(&mut self) -> Option<Self::Item> {
        while let Some(something) = self.dirwalker.next() {
            match something {
                Ok(next) => if next.file_type().is_dir() {
                                return Some(StoreObject::Collection(next.path().to_path_buf()))
                            } else if next.file_type().is_file() {
                                let n   = next.path().to_path_buf();
                                let sid = match StoreId::new(Some(self.store_path.clone()), n) {
                                    Err(e) => {
                                        trace_error(&e);
                                        continue;
                                    },
                                    Ok(o) => o,
                                };
                                return Some(StoreObject::Id(sid))
                            },
                Err(e) => {
                    warn!("Error in Walker");
                    debug!("{:?}", e);
                    return None;
                }
            }
        }

        return None;
    }
}

impl StoreEntry {

    fn new(id: StoreId, backend: &Box<FileAbstraction>) -> Result<StoreEntry> {
        let pb = try!(id.clone().into_pathbuf());

        #[cfg(feature = "fs-lock")]
        {
            try!(open_file(pb.clone())
                .and_then(|f| f.lock_exclusive().map_err_into(SEK::FileError))
                .map_err_into(SEK::IoError));
        }

        Ok(StoreEntry {
            id: id,
            file: backend.new_instance(pb),
            status: StoreEntryStatus::Present,
        })
    }

    /// The entry is currently borrowed, meaning that some thread is currently
    /// mutating it
    fn is_borrowed(&self) -> bool {
        self.status == StoreEntryStatus::Borrowed
    }

    fn get_entry(&mut self) -> Result<Entry> {
        if !self.is_borrowed() {
            self.file
                .get_file_content(self.id.clone())
                .or_else(|err| if err.err_type() == SEK::FileNotFound {
                    Ok(Entry::new(self.id.clone()))
                } else {
                    Err(err)
                })
        } else {
            Err(SE::new(SEK::EntryAlreadyBorrowed, None))
        }
    }

    fn write_entry(&mut self, entry: &Entry) -> Result<()> {
        if self.is_borrowed() {
            assert_eq!(self.id, entry.location);
            self.file.write_file_content(entry)
                .map_err_into(SEK::FileError)
                .map(|_| ())
        } else {
            Ok(())
        }
    }
}

#[cfg(feature = "fs-lock")]
impl Drop for StoreEntry {

    fn drop(self) {
        self.get_entry()
            .and_then(|entry| open_file(entry.get_location().clone()).map_err_into(SEK::IoError))
            .and_then(|f| f.unlock().map_err_into(SEK::FileError))
            .map_err_into(SEK::IoError)
    }

}


/// The Store itself, through this object one can interact with IMAG's entries
pub struct Store {
    location: PathBuf,

    ///
    /// Configuration object of the store
    ///
    configuration: Option<Value>,

    ///
    /// Internal Path->File cache map
    ///
    /// Caches the files, so they remain flock()ed
    ///
    /// Could be optimized for a threadsafe HashMap
    ///
    entries: Arc<RwLock<HashMap<StoreId, StoreEntry>>>,

    /// The backend to use
    ///
    /// This provides the filesystem-operation functions (or pretends to)
    backend: Box<FileAbstraction>,
}

impl Store {

    /// Create a new Store object
    ///
    /// This opens a Store in `location` using the configuration from `store_config` (if absent, it
    /// uses defaults).
    ///
    /// If the configuration is not valid, this fails.
    ///
    /// If the location does not exist, creating directories is by default denied and the operation
    /// fails, if not configured otherwise.
    /// An error is returned in this case.
    ///
    /// If the path exists and is a file, the operation is aborted as well, an error is returned.
    ///
    /// # Return values
    ///
    /// - On success: Store object
    /// - On Failure:
    ///   - ConfigurationError if config is faulty
    ///   - IoError(FileError(CreateStoreDirDenied())) if store location does not exist and creating
    ///     is denied
    ///   - StorePathCreate(_) if creating the store directory failed
    ///   - StorePathExists() if location exists but is a file
    pub fn new(location: PathBuf, store_config: Option<Value>) -> Result<Store> {
        let backend = Box::new(FSFileAbstraction::new());
        Store::new_with_backend(location, store_config, backend)
    }

    /// Create a Store object as descripbed in `Store::new()` documentation, but with an alternative
    /// backend implementation.
    ///
    /// Do not use directly, only for testing purposes.
    pub fn new_with_backend(location: PathBuf,
                            store_config: Option<Value>,
                            backend: Box<FileAbstraction>) -> Result<Store> {
        use configuration::*;

        debug!("Validating Store configuration");
        let _ = try!(config_is_valid(&store_config).map_err_into(SEK::ConfigurationError));

        debug!("Building new Store object");
        if !location.exists() {
            if !config_implicit_store_create_allowed(store_config.as_ref()) {
                warn!("Implicitely creating store directory is denied");
                warn!(" -> Either because configuration does not allow it");
                warn!(" -> or because there is no configuration");
                return Err(SEK::CreateStoreDirDenied.into_error())
                    .map_err_into(SEK::FileError)
                    .map_err_into(SEK::IoError);
            }

            try!(backend.create_dir_all(&location)
                 .map_err_into(SEK::StorePathCreate)
                 .map_dbg_err_str("Failed"));
        } else if location.is_file() {
            debug!("Store path exists as file");
            return Err(SEK::StorePathExists.into_error());
        }

        let store = Store {
            location: location.clone(),
            configuration: store_config,
            entries: Arc::new(RwLock::new(HashMap::new())),
            backend: backend,
        };

        debug!("Store building succeeded");
        debug!("------------------------");
        debug!("{:?}", store);
        debug!("------------------------");

        Ok(store)
    }

    /// Reset the backend of the store during runtime
    ///
    /// # Warning
    ///
    /// This is dangerous!
    /// You should not be able to do that in application code, only the libimagrt should be used to
    /// do this via safe and careful wrapper functions!
    ///
    /// If you are able to do this without using `libimagrt`, please file an issue report.
    ///
    /// # Purpose
    ///
    /// With the I/O backend of the store, the store is able to pipe itself out via (for example)
    /// JSON. But because we need a functionality where we load contents from the filesystem and
    /// then pipe it to stdout, we need to be able to replace the backend during runtime.
    ///
    /// This also applies the other way round: If we get the store from stdin and have to persist it
    /// to stdout, we need to be able to replace the in-memory backend with the real filesystem
    /// backend.
    ///
    /// # TODO
    ///
    /// Currently, this is the naive implementatoin which does not transfer contents of the
    /// backends.
    ///
    pub fn reset_backend(&mut self, mut backend: Box<FileAbstraction>) -> Result<()> {
        self.backend
            .drain()
            .and_then(|drain| backend.fill(drain))
            .map(|_| self.backend = backend)
    }

    /// Get the store configuration
    pub fn config(&self) -> Option<&Value> {
        self.configuration.as_ref()
    }

    /// Verify the store.
    ///
    /// This function is not intended to be called by normal programs but only by `imag-store`.
    #[cfg(feature = "verify")]
    pub fn verify(&self) -> bool {
        use libimagerror::trace::trace_error_dbg;

        info!("Header | Content length | Path");
        info!("-------+----------------+-----");

        WalkDir::new(self.location.clone())
            .into_iter()
            .all(|res| match res {
                Ok(dent) => {
                    if dent.file_type().is_file() {
                        match self.get(PathBuf::from(dent.path())) {
                            Ok(Some(fle)) => {<commit_msg>Remove TODO from Store::reset_backend()<commit_after>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 <NAME> <<EMAIL>> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use std::collections::HashMap;
use std::ops::Drop;
use std::path::PathBuf;
use std::result::Result as RResult;
use std::sync::Arc;
use std::sync::RwLock;
use std::io::Read;
use std::convert::Into;
use std::ops::Deref;
use std::ops::DerefMut;
use std::fmt::Formatter;
use std::fmt::Debug;
use std::fmt::Error as FMTError;

use toml::Value;
use glob::glob;
use walkdir::WalkDir;
use walkdir::Iter as WalkDirIter;

use error::{StoreError as SE, StoreErrorKind as SEK};
use error::MapErrInto;
use storeid::{IntoStoreId, StoreId, StoreIdIterator};
use file_abstraction::FileAbstractionInstance;
use toml_ext::*;
// We re-export the following things so tests can use them
pub use file_abstraction::FileAbstraction;
pub use file_abstraction::FSFileAbstraction;
pub use file_abstraction::InMemoryFileAbstraction;

use libimagerror::into::IntoError;
use libimagerror::trace::trace_error;
use libimagutil::debug_result::*;

use self::glob_store_iter::*;

/// The Result Type returned by any interaction with the store that could fail
pub type Result<T> = RResult<T, SE>;


#[derive(Debug, PartialEq)]
enum StoreEntryStatus {
    Present,
    Borrowed
}

/// A store entry, depending on the option type it is either borrowed currently
/// or not.
#[derive(Debug)]
struct StoreEntry {
    id: StoreId,
    file: Box<FileAbstractionInstance>,
    status: StoreEntryStatus,
}

pub enum StoreObject {
    Id(StoreId),
    Collection(PathBuf),
}

pub struct Walk {
    store_path: PathBuf,
    dirwalker: WalkDirIter,
}

impl Walk {

    fn new(mut store_path: PathBuf, mod_name: &str) -> Walk {
        let pb = store_path.clone();
        store_path.push(mod_name);
        Walk {
            store_path: pb,
            dirwalker: WalkDir::new(store_path).into_iter(),
        }
    }
}

impl ::std::ops::Deref for Walk {
    type Target = WalkDirIter;

    fn deref(&self) -> &Self::Target {
        &self.dirwalker
    }
}

impl Iterator for Walk {
    type Item = StoreObject;

    fn next(&mut self) -> Option<Self::Item> {
        while let Some(something) = self.dirwalker.next() {
            match something {
                Ok(next) => if next.file_type().is_dir() {
                                return Some(StoreObject::Collection(next.path().to_path_buf()))
                            } else if next.file_type().is_file() {
                                let n   = next.path().to_path_buf();
                                let sid = match StoreId::new(Some(self.store_path.clone()), n) {
                                    Err(e) => {
                                        trace_error(&e);
                                        continue;
                                    },
                                    Ok(o) => o,
                                };
                                return Some(StoreObject::Id(sid))
                            },
                Err(e) => {
                    warn!("Error in Walker");
                    debug!("{:?}", e);
                    return None;
                }
            }
        }

        return None;
    }
}

impl StoreEntry {

    fn new(id: StoreId, backend: &Box<FileAbstraction>) -> Result<StoreEntry> {
        let pb = try!(id.clone().into_pathbuf());

        #[cfg(feature = "fs-lock")]
        {
            try!(open_file(pb.clone())
                .and_then(|f| f.lock_exclusive().map_err_into(SEK::FileError))
                .map_err_into(SEK::IoError));
        }

        Ok(StoreEntry {
            id: id,
            file: backend.new_instance(pb),
            status: StoreEntryStatus::Present,
        })
    }

    /// The entry is currently borrowed, meaning that some thread is currently
    /// mutating it
    fn is_borrowed(&self) -> bool {
        self.status == StoreEntryStatus::Borrowed
    }

    fn get_entry(&mut self) -> Result<Entry> {
        if !self.is_borrowed() {
            self.file
                .get_file_content(self.id.clone())
                .or_else(|err| if err.err_type() == SEK::FileNotFound {
                    Ok(Entry::new(self.id.clone()))
                } else {
                    Err(err)
                })
        } else {
            Err(SE::new(SEK::EntryAlreadyBorrowed, None))
        }
    }

    fn write_entry(&mut self, entry: &Entry) -> Result<()> {
        if self.is_borrowed() {
            assert_eq!(self.id, entry.location);
            self.file.write_file_content(entry)
                .map_err_into(SEK::FileError)
                .map(|_| ())
        } else {
            Ok(())
        }
    }
}

#[cfg(feature = "fs-lock")]
impl Drop for StoreEntry {

    fn drop(self) {
        self.get_entry()
            .and_then(|entry| open_file(entry.get_location().clone()).map_err_into(SEK::IoError))
            .and_then(|f| f.unlock().map_err_into(SEK::FileError))
            .map_err_into(SEK::IoError)
    }

}


/// The Store itself, through this object one can interact with IMAG's entries
pub struct Store {
    location: PathBuf,

    ///
    /// Configuration object of the store
    ///
    configuration: Option<Value>,

    ///
    /// Internal Path->File cache map
    ///
    /// Caches the files, so they remain flock()ed
    ///
    /// Could be optimized for a threadsafe HashMap
    ///
    entries: Arc<RwLock<HashMap<StoreId, StoreEntry>>>,

    /// The backend to use
    ///
    /// This provides the filesystem-operation functions (or pretends to)
    backend: Box<FileAbstraction>,
}

impl Store {

    /// Create a new Store object
    ///
    /// This opens a Store in `location` using the configuration from `store_config` (if absent, it
    /// uses defaults).
    ///
    /// If the configuration is not valid, this fails.
    ///
    /// If the location does not exist, creating directories is by default denied and the operation
    /// fails, if not configured otherwise.
    /// An error is returned in this case.
    ///
    /// If the path exists and is a file, the operation is aborted as well, an error is returned.
    ///
    /// # Return values
    ///
    /// - On success: Store object
    /// - On Failure:
    ///   - ConfigurationError if config is faulty
    ///   - IoError(FileError(CreateStoreDirDenied())) if store location does not exist and creating
    ///     is denied
    ///   - StorePathCreate(_) if creating the store directory failed
    ///   - StorePathExists() if location exists but is a file
    pub fn new(location: PathBuf, store_config: Option<Value>) -> Result<Store> {
        let backend = Box::new(FSFileAbstraction::new());
        Store::new_with_backend(location, store_config, backend)
    }

    /// Create a Store object as descripbed in `Store::new()` documentation, but with an alternative
    /// backend implementation.
    ///
    /// Do not use directly, only for testing purposes.
    pub fn new_with_backend(location: PathBuf,
                            store_config: Option<Value>,
                            backend: Box<FileAbstraction>) -> Result<Store> {
        use configuration::*;

        debug!("Validating Store configuration");
        let _ = try!(config_is_valid(&store_config).map_err_into(SEK::ConfigurationError));

        debug!("Building new Store object");
        if !location.exists() {
            if !config_implicit_store_create_allowed(store_config.as_ref()) {
                warn!("Implicitely creating store directory is denied");
                warn!(" -> Either because configuration does not allow it");
                warn!(" -> or because there is no configuration");
                return Err(SEK::CreateStoreDirDenied.into_error())
                    .map_err_into(SEK::FileError)
                    .map_err_into(SEK::IoError);
            }

            try!(backend.create_dir_all(&location)
                 .map_err_into(SEK::StorePathCreate)
                 .map_dbg_err_str("Failed"));
        } else if location.is_file() {
            debug!("Store path exists as file");
            return Err(SEK::StorePathExists.into_error());
        }

        let store = Store {
            location: location.clone(),
            configuration: store_config,
            entries: Arc::new(RwLock::new(HashMap::new())),
            backend: backend,
        };

        debug!("Store building succeeded");
        debug!("------------------------");
        debug!("{:?}", store);
        debug!("------------------------");

        Ok(store)
    }

    /// Reset the backend of the store during runtime
    ///
    /// # Warning
    ///
    /// This is dangerous!
    /// You should not be able to do that in application code, only the libimagrt should be used to
    /// do this via safe and careful wrapper functions!
    ///
    /// If you are able to do this without using `libimagrt`, please file an issue report.
    ///
    /// # Purpose
    ///
    /// With the I/O backend of the store, the store is able to pipe itself out via (for example)
    /// JSON. But because we need a functionality where we load contents from the filesystem and
    /// then pipe it to stdout, we need to be able to replace the backend during runtime.
    ///
    /// This also applies the other way round: If we get the store from stdin and have to persist it
    /// to stdout, we need to be able to replace the in-memory backend with the real filesystem
    /// backend.
    ///
    pub fn reset_backend(&mut self, mut backend: Box<FileAbstraction>) -> Result<()> {
        self.backend
            .drain()
            .and_then(|drain| backend.fill(drain))
            .map(|_| self.backend = backend)
    }

    /// Get the store configuration
    pub fn config(&self) -> Option<&Value> {
        self.configuration.as_ref()
    }

    /// Verify the store.
    ///
    /// This function is not intended to be called by normal programs but only by `imag-store`.
    #[cfg(feature = "verify")]
    pub fn verify(&self) -> bool {
        use libimagerror::trace::trace_error_dbg;

        info!("Header | Content length | Path");
        info!("-------+----------------+-----");

        WalkDir::new(self.location.clone())
            .into_iter()
            .all(|res| match res {
                Ok(dent) => {
                    if dent.file_type().is_file() {
                        match self.get(PathBuf::from(dent.path())) {
                            Ok(Some(fle)) => {
{'fraction_non_alphanumeric': 0.127313689976587, 'fraction_numerical': 0.0018553695277642797, 'mean_word_length': 1.9759432102011305, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '87624', 'n_tokens_mistral': 6342, 'n_tokens_neox': 6186, 'n_words': 1886}