<commit_before>// Index object
function Index() {
  this.fileContents = undefined;
  this.invertedIndex = {};

  // stop words to ignore on index creation
  this.stopWords = ['a', 'an', 'and', 'as', 'at', 'but', 'by', 'each', 'every',
    'for', 'from', 'her', 'his', 'in', 'into', 'its', 'like', 'my', 'no', 'nor',
    'of', 'off', 'on', 'onto', 'or', 'our', 'out', 'outside', 'over', 'past',
    'since', 'so', 'some', 'than', 'that', 'the', 'their', 'this', 'to', 'up',
    'with'
  ];

  this.readFile = function (filepath) {
    var self = this;
    return fetch(filepath)
      .then(function (response) {
        return response.text();
      }).then(function (response) {
        try {
          self.fileContents = JSON.parse(response);
        } catch (e) {
          console.log(e);
        }

        return response;
      }).catch(function (err) {
        console.log('parsing failed', err);
        throw err;
      });
  };

  this.getIndex = function () {
    var self = this;

    this.fileContents.forEach(function (book, index) {
      for (var key in book) {
        var wordArray = book[key].toLowerCase()
          .replace(/\W+/g, ' ').trim().split(' ');
        wordArray.forEach(function (word) {
          // search for and exclude stop words from index creation
          if (self.stopWords.indexOf(word) === -1) {
            if (self.invertedIndex[word]) {
              var currentValue = self.invertedIndex[word];
              if (currentValue.indexOf(index) === -1) {
                currentValue.push(index);
              }
            } else {
              self.invertedIndex[word] = [index];
            }
          }
        });
      }
    });

    return this.invertedIndex;
  };

  this.searchIndex = function (searchTerms) {
    var self = this;
    var terms = [];
    var searchResults = [];

    // convert all format of input into an array
    if (!Array.isArray(searchTerms)) {
      for (var key in arguments) {
        terms.push(arguments[key]);
      }
    } else {<commit_msg>added comments<commit_after>// Index object
function Index() {
  this.fileContents = undefined;
  this.invertedIndex = {};

  // stop words to ignore on index creation
  this.stopWords = ['a', 'an', 'and', 'as', 'at', 'but', 'by', 'each', 'every',
    'for', 'from', 'her', 'his', 'in', 'into', 'its', 'like', 'my', 'no', 'nor',
    'of', 'off', 'on', 'onto', 'or', 'our', 'out', 'outside', 'over', 'past',
    'since', 'so', 'some', 'than', 'that', 'the', 'their', 'this', 'to', 'up',
    'with'
  ];

  this.readFile = function (filepath) {
    var self = this;
    return fetch(filepath)
      .then(function (response) {
        return response.text();
      }).then(function (response) {
        try {
          self.fileContents = JSON.parse(response);
        } catch (e) {
          console.log(e);
        }

        return response;
      }).catch(function (err) {
        console.log('parsing failed', err);
        throw err;
      });
  };

  this.getIndex = function () {
    var self = this;

    this.fileContents.forEach(function (book, index) {
      for (var key in book) {
        var wordArray = book[key].toLowerCase()
          .replace(/\W+/g, ' ').trim().split(' ');
        wordArray.forEach(function (word) {
          // search for and exclude stop words from index creation
          if (self.stopWords.indexOf(word) === -1) {

            /* if a word occurs more than once in the same document, only push
             its index once. i.e if 'alice' occurs thrice in document 0, the
             result should be {'alice': [0]} 
            */
            if (self.invertedIndex[word]) {
              var currentValue = self.invertedIndex[word];
              if (currentValue.indexOf(index) === -1) {
                currentValue.push(index);
              }
            } else {
              self.invertedIndex[word] = [index];
            }
          }
        });
      }
    });

    return this.invertedIndex;
  };

  this.searchIndex = function (searchTerms) {
    var self = this;
    var terms = [];
    var searchResults = [];

    // convert all format of input into an array
    if (!Array.isArray(searchTerms)) {
      for (var key in arguments) {
        terms.push(arguments[key]);
      }
    } else {
{'fraction_non_alphanumeric': 0.16802991353119887, 'fraction_numerical': 0.0014021967749474175, 'mean_word_length': 2.0440967283072546, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1319314', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1295, 'n_words': 349}