<commit_before>/**
* @license Apache-2.0
*
* Copyright (c) 2018 The Stdlib Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

'use strict';

// MODULES //

var logger = require( 'debug' );
var lunr = require( 'lunr' );
var sectionText = require( './section_text.js' );


// VARIABLES //

var debug = logger( 'seach:pkg-index:create' );
var TITLE_REGEXP = /#[ ]*([^\n]+)\r?\n/;
var DESCR_REGEXP = />[ ]*([^\n]+)\r?\n/;


// MAIN //

/**
* Returns a serialized lunr.js search index.
*
* @private
* @param {(EmptyArray|StringArray)} files - list of files and their associated contents
* @returns {Object} serialized search index
*/
function createIndex( files ) {
	var idx = lunr( initialize );
	return idx.toJSON();

	/**
	* Initializes a search index.
	*
	* @private
	*/
	function initialize() {
		/* eslint-disable no-invalid-this */
		var file;
		var blob;
		var i;

		this.field( 'title', {
			'boost': 10
		});
		this.field( 'description', {
			'boost': 5
		});
		this.field( 'intro', {
			'boost': 3
		});
		this.field( 'usage', {
			'boost': 3
		});
		this.field( 'notes', {
			'boost': 2
		} );
		this.field( 'cli', {
			'boost': 1
		});
		this.field( 'references', {
			'boost': 1
		});
		this.ref( 'id' );

		// Add documents to search index:
		for ( i = 0; i < files.length; i++ ) {
			debug( 'Indexing file: %s', files[ i ].file );
			file = files[ i ].data;
			blob = {
				'title': file.match( TITLE_REGEXP )[ 1 ],
				'description': file.match( DESCR_REGEXP )[ 1 ],
				'intro': sectionText( file, 'intro' ),
				'usage': sectionText( file, 'usage' ),
				'notes': sectionText( file, 'notes' ),
				'cli': sectionText( file, 'cli' ),
				'references': sectionText( file, 'references' ),
				'id': files[ i ].file
			};
			debug( 'File data: %s', JSON.stringify( blob ) );<commit_msg>Limit sections which are indexed and lowercase title<commit_after>/**
* @license Apache-2.0
*
* Copyright (c) 2018 The Stdlib Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

'use strict';

// MODULES //

var logger = require( 'debug' );
var lunr = require( 'lunr' );
var lowercase = require( '@stdlib/string/lowercase' );
var sectionText = require( './section_text.js' );


// VARIABLES //

var debug = logger( 'seach:pkg-index:create' );
var TITLE_REGEXP = /^#[ ]*([^\n]+)\r?\n$/;
var DESCR_REGEXP = />[ ]*([^\n]+)\r?\n/;


// MAIN //

/**
* Returns a serialized lunr.js search index.
*
* @private
* @param {(EmptyArray|StringArray)} files - list of files and their associated contents
* @returns {Object} serialized search index
*/
function createIndex( files ) {
	var idx = lunr( initialize );
	return idx.toJSON();

	/**
	* Initializes a search index.
	*
	* @private
	*/
	function initialize() {
		/* eslint-disable no-invalid-this */
		var title;
		var file;
		var blob;
		var desc;
		var i;

		this.field( 'title', {
			'boost': 10
		});
		this.field( 'description', {
			'boost': 5
		});
		this.field( 'introduction', {
			'boost': 3
		});
		this.ref( 'id' );

		// Add documents to search index:
		for ( i = 0; i < files.length; i++ ) {
			debug( 'Indexing file: %s', files[ i ].file );
			file = files[ i ].data;

			title = lowercase( file.match( TITLE_REGEXP )[ 1 ] );
			debug( 'Title: %s', title );

			desc = file.match( DESCR_REGEXP )[ 1 ];
			debug( 'Description: %s', desc );

			blob = {
				'title': title,
				'description': desc,
				'introduction': sectionText( file, 'intro' ),
				'id': files[ i ].file
			};
			debug( 'File data: %s', JSON.stringify( blob ) );
{'fraction_non_alphanumeric': 0.1642952423299244, 'fraction_numerical': 0.008448199199644287, 'mean_word_length': 3.236346516007533, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4331789', 'n_tokens_mistral': 1672, 'n_tokens_neox': 1500, 'n_words': 498}