<commit_before>// * phrase search 
// * html teaser with tagged match

module.exports = function (options) {

    var SearchIndex = require('search-index');
    var path = require('path');

    var preparer = require('./Preparer.js');
    var cfi = require('./CFI.js');

    var indexDB = 'indexDB'; // path to index-db 
    var defaultOption = {'indexPath': indexDB, logLevel: 'warn'};
    options = options || defaultOption;

    var si = new SearchIndex(options);
    var that = this;

    this.indexing = function (pathToEpubs, callback) {

        console.log("normalize epub content");

        path.normalize(pathToEpubs);

        preparer.normalize(pathToEpubs, function (dataSet) {

            console.log("start indexing");
            that.add(dataSet, function (err) {
                //console.log(dataSet);
                if (callback) {
                    if (err)
                        callback(err);
                    else
                        callback('all is indexed');
                }
            });
            //callback('all is indexed');
        });
    };

    this.add = function (jsonDoc, callback) {

        var options = {};
        options.filters = [];
        
        //console.debug(si.indexes);
        //si.get(jsonDoc.epubTitle, function (err, doc) {
        // // update doc with latest version ???
        //});
        
        si.add(options, jsonDoc, callback);
    };

    this.search = function (q, epubTitle, callback) {

        epubTitle = epubTitle || '*'; // if epubTitle undefined return all hits

        // q is an array !!!
        var query = {
            "query": {"*": q},
            "offset": 0,
            "pageSize": 100
        };

        si.search(query, function (err, result) {

            if (result.hits) {

                var hits = [];
                for (var i in result.hits) {

                    var title = result.hits[i].document.epubTitle;
                    if (title === epubTitle || epubTitle === '*') {

                        var data = {
                            "query": q,
                            "spineItemPath": result.hits[i].document.spineItemPath,
                            "baseCfi": result.hits[i].document.baseCfi
                        }

                        var cfiList = cfi.generate(data); 
                        
                        if (cfiList.length > 0) {
                            result.hits[i].document.cfis = cfiList;
                            delete result.hits[i].document.body;
                            delete result.hits[i].document.title;
                            delete result.hits[i].document['*'];
                            delete result.hits[i].document.spineItemPath;

                            hits.push(result.hits[i].document);
                        }
                    }
                }
                callback(hits);
            }
        })
    };

    this.match = function (beginsWith, callback) {
        si.match(beginsWith, callback);
    };

    this.empty = function (callback) {
        si.empty(callback);
    };
};<commit_msg>- added contract to check epub data folder is not empty - added close()<commit_after>// * phrase search 
// * html teaser with tagged match

module.exports = function (options) {

    var SearchIndex = require('search-index');
    var path = require('path');
    var fs = require('extfs');

    var preparer = require('./Preparer.js');
    var cfi = require('./CFI.js');

    var indexDB = 'indexDB'; // path to index-db 
    var defaultOption = {'indexPath': indexDB, logLevel: 'warn'};
    options = options || defaultOption;

    var si = new SearchIndex(options);
    var that = this;

    this.indexing = function (pathToEpubs, callback) {

        if(fs.isEmptySync(pathToEpubs)) 
          return callback(new Error('Can`t index empty folder: ' + pathToEpubs));
        
        console.log("normalize epub content");
        
        path.normalize(pathToEpubs);

        preparer.normalize(pathToEpubs, function (dataSet) {

            console.log("start indexing");
            that.add(dataSet, function (err) {
                //console.log(dataSet);
                if (callback) {
                    if (err)
                        callback(err);
                    else
                        callback('all is indexed');
                }
            });
            //callback('all is indexed');
        });
    };

    this.add = function (jsonDoc, callback) {

        var options = {};
        options.filters = [];
        
        //console.debug(si.indexes);
        //si.get(jsonDoc.epubTitle, function (err, doc) {
        // // update doc with latest version ???
        //});
        
        si.add(options, jsonDoc, callback);
    };

    this.search = function (q, epubTitle, callback) {

        epubTitle = epubTitle || '*'; // if epubTitle undefined return all hits

        // q is an array !!!
        var query = {
            "query": {"*": q},
            "offset": 0,
            "pageSize": 100
        };

        si.search(query, function (err, result) {

            if (result.hits) {

                var hits = [];
                for (var i in result.hits) {

                    var title = result.hits[i].document.epubTitle;
                    if (title === epubTitle || epubTitle === '*') {

                        var data = {
                            "query": q,
                            "spineItemPath": result.hits[i].document.spineItemPath,
                            "baseCfi": result.hits[i].document.baseCfi
                        }

                        var cfiList = cfi.generate(data); 
                        
                        if (cfiList.length > 0) {
                            result.hits[i].document.cfis = cfiList;
                            delete result.hits[i].document.body;
                            delete result.hits[i].document.title;
                            delete result.hits[i].document['*'];
                            delete result.hits[i].document.spineItemPath;

                            hits.push(result.hits[i].document);
                        }
                    }
                }
                callback(hits);
            }
        })
    };

    this.match = function (beginsWith, callback) {
        si.match(beginsWith, callback);
    };

    this.empty = function (callback) {
        si.empty(callback);
    };
    
    this.close = function(callback) {
        si.close(callback);
    }
};
{'fraction_non_alphanumeric': 0.12450351359608922, 'fraction_numerical': 0.0015276504735716467, 'mean_word_length': 1.2971929824561403, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5679699', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1712, 'n_words': 376}