<commit_before>#!/usr/bin/env node

var optimist = require('optimist');
var Redis = require('ioredis');
var app = require('../lib/app');
var fs = require('fs');
var myUtils = require('../lib/util');

var redisConnections = [];

var args = optimist
  .alias('h', 'help')
  .alias('h', '?')
  .options('redis-port', {
    string: true,
    describe: 'The port to find redis on.'
  })
  .options('sentinel-port', {
    string: true,
    describe: 'The port to find sentinel on.'
  })
  .options('redis-host', {
    string: true,
    describe: 'The host to find redis on.'
  })
  .options('sentinel-host', {
    string: true,
    describe: 'The host to find sentinel on.'
  })
  .options('redis-socket', {
    string: true,
    describe: 'The unix-socket to find redis on.'
  })
  .options('redis-password', {
    string: true,
    describe: 'The redis password.'
  })
  .options('redis-db', {
    string: true,
    describe: 'The redis database.'
  })
  .options('http-auth-username', {
    alias: "http-u",
    string: true,
    describe: 'The http authorisation username.'
  })
  .options('http-auth-password', {
    alias: "http-p",
    string: true,
    describe: 'The http authorisation password.'
  })
  .options('http-auth-password-hash', {
    alias: "http-h",
    string: true,
    describe: 'The http authorisation password hash.'
  })
  .options('address', {
    alias: 'a',
    string: true,
    describe: 'The address to run the server on.',
    default: "0.0.0.0"
  })
  .options('port', {
    alias: 'p',
    string: true,
    describe: 'The port to run the server on.',
    default: 8081
  })
  .options('url-prefix', {
    alias: 'u',
    string: true,
    describe: 'The url prefix to respond on.',
    default: '',
  })
  .options('nosave', {
    alias: 'ns',
    boolean: true,
    describe: 'Do not save new connections to config.',
    default: false
  })
  .options('noload', {
    alias: 'nl',
    boolean: true,
    describe: 'Do not load connections from config.'
  })
  .options('clear-config', {
    alias: 'cc',
    boolean: false,
    describe: 'clear configuration file'
  })
  .options('root-pattern', {
      alias: 'rp',
      boolean: false,
      describe: 'default root pattern for redis keys',
      default: '*'
  })
  .alias('sc', 'use-scan')
  .argv;

if (args.help) {
  optimist.showHelp();
  return process.exit(-1);
}

if (args['use-scan']) {
  console.log('Using scan instead of keys');
  Object.defineProperty(Redis.prototype, 'keys', {
    value: function(pattern, cb) {
      var keys = [];
      var scanCB = function(err, res) {
        if (err) {
          cb(err);
        } else {
          var count = res[0], curKeys = res[1];
          keys = keys.concat(curKeys);
          if (Number(count) === 0) {
            cb(null, keys);
          } else {
            this.scan(pattern, count, scanCB);
          }
        }
      };
      return this.scan(0, 'MATCH', pattern, scanCB);
    }
  });
}

if(args['clear-config']) {
  myUtils.deleteConfig(function(err) {
    if (err) {
      console.log("Failed to delete existing config file.");
    }
  });
}

myUtils.getConfig(function (err, config) {
  if (err) {
    console.dir(err);
    console.log("No config found or was invalid.\nUsing default configuration.");
    config = {
      "sidebarWidth": 250,<commit_msg>fix scan, add option to specify scan count<commit_after>#!/usr/bin/env node

var optimist = require('optimist');
var Redis = require('ioredis');
var app = require('../lib/app');
var fs = require('fs');
var myUtils = require('../lib/util');

var redisConnections = [];

var args = optimist
  .alias('h', 'help')
  .alias('h', '?')
  .options('redis-port', {
    string: true,
    describe: 'The port to find redis on.'
  })
  .options('sentinel-port', {
    string: true,
    describe: 'The port to find sentinel on.'
  })
  .options('redis-host', {
    string: true,
    describe: 'The host to find redis on.'
  })
  .options('sentinel-host', {
    string: true,
    describe: 'The host to find sentinel on.'
  })
  .options('redis-socket', {
    string: true,
    describe: 'The unix-socket to find redis on.'
  })
  .options('redis-password', {
    string: true,
    describe: 'The redis password.'
  })
  .options('redis-db', {
    string: true,
    describe: 'The redis database.'
  })
  .options('http-auth-username', {
    alias: "http-u",
    string: true,
    describe: 'The http authorisation username.'
  })
  .options('http-auth-password', {
    alias: "http-p",
    string: true,
    describe: 'The http authorisation password.'
  })
  .options('http-auth-password-hash', {
    alias: "http-h",
    string: true,
    describe: 'The http authorisation password hash.'
  })
  .options('address', {
    alias: 'a',
    string: true,
    describe: 'The address to run the server on.',
    default: "0.0.0.0"
  })
  .options('port', {
    alias: 'p',
    string: true,
    describe: 'The port to run the server on.',
    default: 8081
  })
  .options('url-prefix', {
    alias: 'u',
    string: true,
    describe: 'The url prefix to respond on.',
    default: '',
  })
  .options('nosave', {
    alias: 'ns',
    boolean: true,
    describe: 'Do not save new connections to config.',
    default: false
  })
  .options('noload', {
    alias: 'nl',
    boolean: true,
    describe: 'Do not load connections from config.'
  })
  .options('clear-config', {
    alias: 'cc',
    boolean: false,
    describe: 'clear configuration file'
  })
  .options('root-pattern', {
      alias: 'rp',
      boolean: false,
      describe: 'default root pattern for redis keys',
      default: '*'
  })
  .options('use-scan', {
      alias: 'sc',
      boolean: true,
      default: false,
      describe: 'Use SCAN instead of KEYS'
  })
  .options('scan-count', {
      boolean: false,
      default: 100,
      describe: 'The size of each seperate scan'
  })
  .argv;

if (args.help) {
  optimist.showHelp();
  return process.exit(-1);
}

if (args['use-scan']) {
  console.log('Using scan instead of keys');
  Object.defineProperty(Redis.prototype, 'keys', {
    value: function(pattern, cb) {
      var keys = [];
      var that = this;
      var scanCB = function(err, res) {
        if (err) {
          cb(err);
        } else {
          var count = res[0], curKeys = res[1];
	  console.log("scanning: " + count + ": " + curKeys.length);
          keys = keys.concat(curKeys);
          if (Number(count) === 0) {
            cb(null, keys);
          } else {
            that.scan(count, 'MATCH', pattern, 'COUNT', args['scan-count'], scanCB);
          }
        }
      };
      return this.scan(0, 'MATCH', pattern, 'COUNT', args['scan-count'], scanCB);
    }
  });
}

if(args['clear-config']) {
  myUtils.deleteConfig(function(err) {
    if (err) {
      console.log("Failed to delete existing config file.");
    }
  });
}

myUtils.getConfig(function (err, config) {
  if (err) {
    console.dir(err);
    console.log("No config found or was invalid.\nUsing default configuration.");
    config = {
      "sidebarWidth": 250,
{'fraction_non_alphanumeric': 0.16870034052213395, 'fraction_numerical': 0.004965947786606129, 'mean_word_length': 2.5422110552763817, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '475609', 'n_tokens_mistral': 2361, 'n_tokens_neox': 2326, 'n_words': 620}