<commit_before>// Copyright 2015-2018 <NAME>. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

use cortex::backend::Backend;
use cortex::models::Corpus;
use regex::Regex;
use std::env;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Read;
use Archive::*;

/// Reads a lit of arXiv ids given on input, and packages the respective `CorTeX` entries into a
/// new sandbox.
fn main() {
  // Read input arguments
  let mut input_args = env::args();
  let _ = input_args.next(); // skip process name
  let ids_filepath = match input_args.next() {
    Some(path) => path,
    None => "arxiv_ids.txt".to_string(),
  };
  let sandbox_path = match input_args.next() {
    Some(path) => path,
    None => "sandbox.zip".to_string(),
  };

  // Fish out the arXiv root directory from CorTeX
  let backend = Backend::default();
  let sandbox_start = time::get_time();
  let corpus = match Corpus::find_by_name("arXMLiv", &backend.connection) {
    Ok(corpus) => corpus,
    _ => {
      println!("--  The arXMLiv corpus isn't registered in the CorTeX backend, aborting.");
      return;
    },
  };
  let corpus_path = corpus.path;
  println!("-- using arXiv path: {:?}", corpus_path);

  // Prepare to read in the arXiv ids we will sandbox
  let ids_fh = match File::open(&ids_filepath) {
    Ok(fh) => fh,
    _ => {
      println!(
        "-- Couldn't read file with arXiv ids {:?}, aborting.",
        ids_filepath
      );
      return;
    },
  };

  // Prepare a sandbox archive file writer
  let mut sandbox_writer = Writer::new()
    .unwrap()<commit_msg>lowercase name<commit_after>// Copyright 2015-2018 <NAME>. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

use cortex::backend::Backend;
use cortex::models::Corpus;
use regex::Regex;
use std::env;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Read;
use Archive::*;

/// Reads a lit of arXiv ids given on input, and packages the respective `CorTeX` entries into a
/// new sandbox.
fn main() {
  // Read input arguments
  let mut input_args = env::args();
  let _ = input_args.next(); // skip process name
  let ids_filepath = match input_args.next() {
    Some(path) => path,
    None => "arxiv_ids.txt".to_string(),
  };
  let sandbox_path = match input_args.next() {
    Some(path) => path,
    None => "sandbox.zip".to_string(),
  };

  // Fish out the arXiv root directory from CorTeX
  let backend = Backend::default();
  let sandbox_start = time::get_time();
  let corpus = match Corpus::find_by_name("arxmliv", &backend.connection) {
    Ok(corpus) => corpus,
    _ => {
      println!("--  The arXMLiv corpus isn't registered in the CorTeX backend, aborting.");
      return;
    },
  };
  let corpus_path = corpus.path;
  println!("-- using arXiv path: {:?}", corpus_path);

  // Prepare to read in the arXiv ids we will sandbox
  let ids_fh = match File::open(&ids_filepath) {
    Ok(fh) => fh,
    _ => {
      println!(
        "-- Couldn't read file with arXiv ids {:?}, aborting.",
        ids_filepath
      );
      return;
    },
  };

  // Prepare a sandbox archive file writer
  let mut sandbox_writer = Writer::new()
    .unwrap()
{'fraction_non_alphanumeric': 0.14417435037720033, 'fraction_numerical': 0.0044705224923162895, 'mean_word_length': 3.211764705882353, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4521018', 'n_tokens_mistral': 1199, 'n_tokens_neox': 1118, 'n_words': 364}