<commit_before>/**
 * @module myModule
 * @summary: This module's purpose is to:
 *
 * @description:
 *
 * Author: <NAME>
 * Created On: 2015-04-11.
 * @license Apache-2.0
 */

"use strict";

module.exports = function construct(config, dal, Storage) {
  var m = {};
  config = config || {};
  config = _.defaults(config, {});

  /**
   * @description
   * It is import to implement a query method on the provided "dal".  Here is an example
   * of how it is called by this module:
   * Ex. dal.query(params.indexName, { limit: 1, sortBy: 'id', sortDirection: 'desc' })
   *
   * It automatically adds the following properties to index rows:
   * arrivedOn (which is the LastModified data according to S3)
   * key (which is the key to the file on s3)
   *
   * @notes For internal housekeeping purposes the index requires a "key" and "indexOn" column in the index.
   * @param indexName
   * @param bucket
   * @param parser The parser must use "key" as the name of the column for the key's name.
   */
  m.runIndexer = function(params) {
    var bucket = Storage(params.bucketName);
    return dal.query(params.indexName, { limit: 1, sortBy: 'id', sortDirection: 'desc' })
      .then(function(rows) {
        if (rows && rows.length) {
          return rows[0].key;
        }
        return null;
      })
      .then(function(lastKey) {
        return bucket.list(null, lastKey)
          .then(function(objects) {
            log('New Object Count From S3:', objects.length);
            return p.map(objects, function(object) {
              return bucket.readString(object.Key).then(function(data) {
                return {
                  key: object.Key,
                  data: data,<commit_msg>Changed indexer to perform work in chunks of blob lists by 1000<commit_after>/**
 * @module myModule
 * @summary: This module's purpose is to:
 *
 * @description:
 *
 * Author: <NAME>
 * Created On: 2015-04-11.
 * @license Apache-2.0
 */

"use strict";

module.exports = function construct(config, dal, Storage) {
  var m = {};
  config = config || {};
  config = _.defaults(config, {});

  /**
   * @description
   * It is import to implement a query method on the provided "dal".  Here is an example
   * of how it is called by this module:
   * Ex. dal.query(params.indexName, { limit: 1, sortBy: 'id', sortDirection: 'desc' })
   *
   * It automatically adds the following properties to index rows:
   * arrivedOn (which is the LastModified data according to S3)
   * key (which is the key to the file on s3)
   *
   * @notes For internal housekeeping purposes the index requires a "key" and "indexOn" column in the index.
   * @param indexName
   * @param bucket
   * @param parser The parser must use "key" as the name of the column for the key's name.
   */
  m.runIndexer = function(params) {
    var bucket = Storage(params.bucketName);
    return dal.query(params.indexName, { limit: 1, sortBy: 'id', sortDirection: 'desc' })
      .then(function(rows) {
        if (rows && rows.length) {
          return rows[0].key;
        }
        return null;
      })
      .then(function(lastKey) {
        return bucket.list(null, lastKey, 1000)
          .then(function(objects) {
            log('New Object Count From S3:', objects.length);
            return p.map(objects, function(object) {
              return bucket.readString(object.Key).then(function(data) {
                return {
                  key: object.Key,
                  data: data,
{'fraction_non_alphanumeric': 0.11959654178674352, 'fraction_numerical': 0.011527377521613832, 'mean_word_length': 2.531027466937945, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '64904', 'n_tokens_mistral': 1053, 'n_tokens_neox': 1016, 'n_words': 351}