<commit_before>{
  "controls": {
    "mongodbReadCapacity": 10000,
    "elasticsearchBulkSize": 5000,
    "tailFromTime": "2017-05-12T07:47:16.030Z",
    "indexNameSuffix": "_v1"
  },
  "mongodb": {
    "url": "mongodb://user:password@localhost:27017/db?replicaSet=rs0",
    "options": {
      "authSource": "admin",
      "poolSize": 3,
      "readPreference": "secondaryPreferred"
    }
  },
  "elasticsearch": {
    "options": {
      "host": "http://localhost:9200",
      "apiVersion": "5.0",
      "sniffOnStart": true,
      "sniffOnConnectionFault": true,
      "sniffInterval": 60
    },
    "indices": [
      {
        "index": "index0",
        "body": {
          "settings": {
            "index": {
              "number_of_shards": 3,
              "number_of_replicas": 1,
              "mapper.dynamic": false
            }
          }
        }
      }
    ]
  },
  "tasks": [
    {
      "extract": {
        "db": "db0",
        "collection": "collection0",
        "query": {
          "createdAt": {
            "gte": 1494000000000
          }
        },
        "projection": {
          "field0": 1,
          "field1": 1,
          "field2": 1<commit_msg>update config<commit_after>{
  "controls": {
    "mongodbReadCapacity": 10000,
    "elasticsearchBulkSize": 5000,
    "tailFromTime": "2017-05-12T07:47:16.030Z",
    "indexNameSuffix": "_v1"
  },
  "mongodb": {
    "url": "mongodb://user:password@localhost:27017/db?replicaSet=rs0",
    "options": {
      "authSource": "admin",
      "poolSize": 3,
      "readPreference": "secondaryPreferred"
    }
  },
  "elasticsearch": {
    "options": {
      "host": "http://localhost:9200",
      "apiVersion": "5.2"
    },
    "indices": [
      {
        "index": "index0",
        "body": {
          "settings": {
            "index": {
              "number_of_shards": 3,
              "number_of_replicas": 1,
              "mapper.dynamic": false
            }
          }
        }
      }
    ]
  },
  "tasks": [
    {
      "extract": {
        "db": "db0",
        "collection": "collection0",
        "query": {
          "createdAt": {
            "gte": 1494000000000
          }
        },
        "projection": {
          "field0": 1,
          "field1": 1,
          "field2": 1
{'fraction_non_alphanumeric': 0.17525319242624396, 'fraction_numerical': 0.057243505063848524, 'mean_word_length': 1.6604215456674474, 'pattern_counts': {'":': 71, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5659856', 'n_tokens_mistral': 852, 'n_tokens_neox': 769, 'n_words': 116}