<commit_before>extern crate rustc_serialize;
extern crate unicode_segmentation;
#[macro_use]
extern crate log;
#[macro_use]
extern crate maplit;
extern crate chrono;
extern crate roaring;
extern crate byteorder;

pub mod term;
pub mod token;
pub mod analysis;
pub mod document;
pub mod store;
pub mod similarity;
pub mod query;
pub mod query_set;
pub mod request;
pub mod response;
<commit_msg>Reexport commonly used public API<commit_after>extern crate rustc_serialize;
extern crate unicode_segmentation;
#[macro_use]
extern crate log;
#[macro_use]
extern crate maplit;
extern crate chrono;
extern crate roaring;
extern crate byteorder;

pub mod term;
pub mod token;
pub mod analysis;
pub mod document;
pub mod store;
pub mod similarity;
pub mod query;
pub mod query_set;
pub mod request;
pub mod response;

pub use term::Term;
pub use token::Token;
pub use analysis::Analyzer;
pub use document::Document;
pub use query::term_matcher::TermMatcher;
pub use query::term_scorer::TermScorer;
pub use query::Query;

{'fraction_non_alphanumeric': 0.076162215628091, 'fraction_numerical': 0.0, 'mean_word_length': 6.496296296296296, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2589048', 'n_tokens_mistral': 328, 'n_tokens_neox': 309, 'n_words': 131}