<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! High level Hat API


use rustc_serialize::{json};
use rustc_serialize::json::{ToJson};
use std::str;
use std::collections::{BTreeMap};

use process::{Process};

use blob_index::{BlobIndex};
use blob_store::{BlobStore, BlobStoreProcess, BlobStoreBackend};

use hash_index::{Hash, HashIndex, HashIndexProcess};
use key_index::{KeyIndex, KeyEntry};
use key_store::{KeyStore, KeyStoreProcess};
use key_store;
use snapshot_index::{SnapshotIndex, SnapshotIndexProcess};
use snapshot_index;

use hash_tree;
use listdir;

use std::path::PathBuf;
use std::fs;
use std::io;
use std::io::{Read, Write};

use std::sync;
use std::sync::atomic;

use time;


pub struct Hat<B> {
  repository_root: PathBuf,
  snapshot_index: SnapshotIndexProcess,
  blob_store: BlobStoreProcess,
  hash_index: HashIndexProcess,
  blob_backend: B,
  hash_backend: key_store::HashStoreBackend,
  max_blob_size: usize,
}

fn concat_filename(a: &PathBuf, b: String) -> String {
  let mut result = a.clone();
  result.push(&b);
  result.into_os_string().into_string().unwrap()
}

fn snapshot_index_name(root: &PathBuf) -> String {
  concat_filename(root, "snapshot_index.sqlite3".to_string())
}

fn blob_index_name(root: &PathBuf) -> String {
  concat_filename(root, "blob_index.sqlite3".to_string())
}

fn hash_index_name(root: &PathBuf) -> String {
  concat_filename(root, "hash_index.sqlite3".to_string())
}

impl <B: 'static + BlobStoreBackend + Clone + Send> Hat<B> {
  pub fn open_repository(repository_root: &PathBuf, backend: B, max_blob_size: usize)
                        -> Hat<B> {
    let snapshot_index_path = snapshot_index_name(repository_root);
    let blob_index_path = blob_index_name(repository_root);
    let hash_index_path = hash_index_name(repository_root);
    let si_p = Process::new(Box::new(move|| { SnapshotIndex::new(snapshot_index_path) }));
    let bi_p = Process::new(Box::new(move|| { BlobIndex::new(blob_index_path) }));
    let hi_p = Process::new(Box::new(move|| { HashIndex::new(hash_index_path) }));

    let local_blob_index = bi_p.clone();
    let local_backend = backend.clone();
    let bs_p = Process::new(Box::new(move|| {
      BlobStore::new(local_blob_index, local_backend, max_blob_size) }));

    Hat{repository_root: repository_root.clone(),
        snapshot_index: si_p,
        hash_index: hi_p.clone(),
        blob_store: bs_p.clone(),
        blob_backend: backend.clone(),
        hash_backend: key_store::HashStoreBackend::new(hi_p, bs_p),
        max_blob_size: max_blob_size,
    }
  }

  pub fn open_family(&self, name: String) -> Option<Family> {
    // We setup a standard pipeline of processes:
    // KeyStore -> KeyIndex
    //          -> HashIndex
    //          -> BlobStore -> BlobIndex

    let key_index_path = concat_filename(&self.repository_root, name.clone());
    let ki_p = Process::new(Box::new(move|| { KeyIndex::new(key_index_path) }));

    let local_ks = KeyStore::new(ki_p.clone(), self.hash_index.clone(), self.blob_store.clone());
    let ks_p = Process::new(Box::new(move|| { local_ks }));

    let ks = KeyStore::new(ki_p, self.hash_index.clone(), self.blob_store.clone());
    Some(Family{name: name, key_store: ks, key_store_process: ks_p})
  }

  pub fn commit(&self, family_name: String) {
    let mut family = self.open_family(family_name.clone()).expect(
      &format!("Could not open family '{}'", family_name));

    // Commit snapshot:
    let (hash, top_ref) = family.commit();
    family.flush();

    // Update to snapshot index:
    self.snapshot_index.send_reply(snapshot_index::Msg::Add(family_name, hash, top_ref));
    self.snapshot_index.send_reply(snapshot_index::Msg::Flush);
  }

  pub fn checkout_in_dir(&self, family_name: String, output_dir: PathBuf) {
    // Extract latest snapshot info:
    let (dir_hash, dir_ref) =
      match self.snapshot_index.send_reply(snapshot_index::Msg::Latest(family_name.clone())) {
        snapshot_index::Reply::Latest(Some((h, r))) => (h, r),
        snapshot_index::Reply::Latest(None) =>
          panic!("Tries to checkout family '{}' before first commit", family_name),
        _ => panic!("Unexpected result from snapshot index"),
      };

    let family = self.open_family(family_name.clone()).expect(
      &format!("Could not open family '{}'", family_name));

    let mut output_dir = output_dir;
    self.checkout_dir_ref(&family, &mut output_dir, dir_hash, dir_ref);
  }

  fn checkout_dir_ref(&self, family: &Family, output: &mut PathBuf, dir_hash: Hash, dir_ref: Vec<u8>) {
    fs::create_dir_all(&output).unwrap();
    for o in family.fetch_dir_data(dir_hash, dir_ref, self.hash_backend.clone()) {
      for d in o.as_array().unwrap().iter() {
        if let Some(ref m) = d.as_object() {
          let name: Vec<u8> = m.get("name").unwrap().as_array()
            .unwrap().iter().map(|x| x.as_i64().unwrap() as u8).collect();<commit_msg>Minor: fix wording in info message<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! High level Hat API


use rustc_serialize::{json};
use rustc_serialize::json::{ToJson};
use std::str;
use std::collections::{BTreeMap};

use process::{Process};

use blob_index::{BlobIndex};
use blob_store::{BlobStore, BlobStoreProcess, BlobStoreBackend};

use hash_index::{Hash, HashIndex, HashIndexProcess};
use key_index::{KeyIndex, KeyEntry};
use key_store::{KeyStore, KeyStoreProcess};
use key_store;
use snapshot_index::{SnapshotIndex, SnapshotIndexProcess};
use snapshot_index;

use hash_tree;
use listdir;

use std::path::PathBuf;
use std::fs;
use std::io;
use std::io::{Read, Write};

use std::sync;
use std::sync::atomic;

use time;


pub struct Hat<B> {
  repository_root: PathBuf,
  snapshot_index: SnapshotIndexProcess,
  blob_store: BlobStoreProcess,
  hash_index: HashIndexProcess,
  blob_backend: B,
  hash_backend: key_store::HashStoreBackend,
  max_blob_size: usize,
}

fn concat_filename(a: &PathBuf, b: String) -> String {
  let mut result = a.clone();
  result.push(&b);
  result.into_os_string().into_string().unwrap()
}

fn snapshot_index_name(root: &PathBuf) -> String {
  concat_filename(root, "snapshot_index.sqlite3".to_string())
}

fn blob_index_name(root: &PathBuf) -> String {
  concat_filename(root, "blob_index.sqlite3".to_string())
}

fn hash_index_name(root: &PathBuf) -> String {
  concat_filename(root, "hash_index.sqlite3".to_string())
}

impl <B: 'static + BlobStoreBackend + Clone + Send> Hat<B> {
  pub fn open_repository(repository_root: &PathBuf, backend: B, max_blob_size: usize)
                        -> Hat<B> {
    let snapshot_index_path = snapshot_index_name(repository_root);
    let blob_index_path = blob_index_name(repository_root);
    let hash_index_path = hash_index_name(repository_root);
    let si_p = Process::new(Box::new(move|| { SnapshotIndex::new(snapshot_index_path) }));
    let bi_p = Process::new(Box::new(move|| { BlobIndex::new(blob_index_path) }));
    let hi_p = Process::new(Box::new(move|| { HashIndex::new(hash_index_path) }));

    let local_blob_index = bi_p.clone();
    let local_backend = backend.clone();
    let bs_p = Process::new(Box::new(move|| {
      BlobStore::new(local_blob_index, local_backend, max_blob_size) }));

    Hat{repository_root: repository_root.clone(),
        snapshot_index: si_p,
        hash_index: hi_p.clone(),
        blob_store: bs_p.clone(),
        blob_backend: backend.clone(),
        hash_backend: key_store::HashStoreBackend::new(hi_p, bs_p),
        max_blob_size: max_blob_size,
    }
  }

  pub fn open_family(&self, name: String) -> Option<Family> {
    // We setup a standard pipeline of processes:
    // KeyStore -> KeyIndex
    //          -> HashIndex
    //          -> BlobStore -> BlobIndex

    let key_index_path = concat_filename(&self.repository_root, name.clone());
    let ki_p = Process::new(Box::new(move|| { KeyIndex::new(key_index_path) }));

    let local_ks = KeyStore::new(ki_p.clone(), self.hash_index.clone(), self.blob_store.clone());
    let ks_p = Process::new(Box::new(move|| { local_ks }));

    let ks = KeyStore::new(ki_p, self.hash_index.clone(), self.blob_store.clone());
    Some(Family{name: name, key_store: ks, key_store_process: ks_p})
  }

  pub fn commit(&self, family_name: String) {
    let mut family = self.open_family(family_name.clone()).expect(
      &format!("Could not open family '{}'", family_name));

    // Commit snapshot:
    let (hash, top_ref) = family.commit();
    family.flush();

    // Update to snapshot index:
    self.snapshot_index.send_reply(snapshot_index::Msg::Add(family_name, hash, top_ref));
    self.snapshot_index.send_reply(snapshot_index::Msg::Flush);
  }

  pub fn checkout_in_dir(&self, family_name: String, output_dir: PathBuf) {
    // Extract latest snapshot info:
    let (dir_hash, dir_ref) =
      match self.snapshot_index.send_reply(snapshot_index::Msg::Latest(family_name.clone())) {
        snapshot_index::Reply::Latest(Some((h, r))) => (h, r),
        snapshot_index::Reply::Latest(None) =>
          panic!("Tried to checkout family '{}' before first commit", family_name),
        _ => panic!("Unexpected result from snapshot index"),
      };

    let family = self.open_family(family_name.clone()).expect(
      &format!("Could not open family '{}'", family_name));

    let mut output_dir = output_dir;
    self.checkout_dir_ref(&family, &mut output_dir, dir_hash, dir_ref);
  }

  fn checkout_dir_ref(&self, family: &Family, output: &mut PathBuf, dir_hash: Hash, dir_ref: Vec<u8>) {
    fs::create_dir_all(&output).unwrap();
    for o in family.fetch_dir_data(dir_hash, dir_ref, self.hash_backend.clone()) {
      for d in o.as_array().unwrap().iter() {
        if let Some(ref m) = d.as_object() {
          let name: Vec<u8> = m.get("name").unwrap().as_array()
            .unwrap().iter().map(|x| x.as_i64().unwrap() as u8).collect();
{'fraction_non_alphanumeric': 0.14337601016979934, 'fraction_numerical': 0.002905656950876237, 'mean_word_length': 4.23977164605138, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2704434', 'n_tokens_mistral': 3609, 'n_tokens_neox': 3635, 'n_words': 832}