<commit_before>extern crate ammonia;
extern crate elasticlunr;

use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::path::Path;

use pulldown_cmark::*;
use serde_json;
use self::elasticlunr::Index;

use book::{Book, BookItem};
use config::Search;
use errors::*;
use utils;
use theme::searcher;

/// Creates all files required for search.
pub fn create_files(search_config: &Search, destination: &Path, book: &Book) -> Result<()> {
    let mut index = Index::new(&["title", "body", "breadcrumbs"]);
    let mut doc_urls = Vec::with_capacity(book.sections.len());

    for item in book.iter() {
        render_item(&mut index, &search_config, &mut doc_urls, item)?;
    }

    let index = write_to_js(index, &search_config, doc_urls)?;
    debug!("Writing search index ✓");

    if search_config.copy_js {
        utils::fs::write_file(destination, "searchindex.js", index.as_bytes())?;
        utils::fs::write_file(destination, "searcher.js", searcher::JS)?;
        utils::fs::write_file(destination, "mark.min.js", searcher::MARK_JS)?;
        utils::fs::write_file(destination, "elasticlunr.min.js", searcher::ELASTICLUNR_JS)?;<commit_msg>Warn when search index is >10MB<commit_after>extern crate ammonia;
extern crate elasticlunr;

use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::path::Path;

use pulldown_cmark::*;
use serde_json;
use self::elasticlunr::Index;

use book::{Book, BookItem};
use config::Search;
use errors::*;
use utils;
use theme::searcher;

/// Creates all files required for search.
pub fn create_files(search_config: &Search, destination: &Path, book: &Book) -> Result<()> {
    let mut index = Index::new(&["title", "body", "breadcrumbs"]);
    let mut doc_urls = Vec::with_capacity(book.sections.len());

    for item in book.iter() {
        render_item(&mut index, &search_config, &mut doc_urls, item)?;
    }

    let index = write_to_js(index, &search_config, doc_urls)?;
    debug!("Writing search index ✓");
    if index.len() > 10_000_000 {
        warn!("searchindex.js is very large ({} bytes)", index.len());
    }

    if search_config.copy_js {
        utils::fs::write_file(destination, "searchindex.js", index.as_bytes())?;
        utils::fs::write_file(destination, "searcher.js", searcher::JS)?;
        utils::fs::write_file(destination, "mark.min.js", searcher::MARK_JS)?;
        utils::fs::write_file(destination, "elasticlunr.min.js", searcher::ELASTICLUNR_JS)?;
{'fraction_non_alphanumeric': 0.16619144602851324, 'fraction_numerical': 0.004073319755600814, 'mean_word_length': 4.961165048543689, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4654603', 'n_tokens_mistral': 833, 'n_tokens_neox': 842, 'n_words': 189}