<commit_before>{
	"work_queue": {
		"name": "active-data-index",
		"debug": true,
		"$ref": "file://~/private.json#aws_credentials"
	},
	"workers": [
		{
			"name": "unit",
			"batch_size": 1000,
			"skip": 0.95,
			"rollover":{
				"field": "build.date",
				"interval": "week",
				"max": "5week"
			},
			"sample_only": [
			],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "unittest",
				"type": "test_result",
				"timeout": 300,
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"schema": {
					"$ref": "//../../schema/test_results.json"
				},
				"debug": false,
				"limit_replicas": false
			},
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "active-data-test-result",
				"key_format": "t.a:b.c"
			}
		},
		{
			"name": "perf",
			"rollover":{
				"field": "build.date",
				"interval": "year",
				"max":"2year"
			},
			"batch_size": 1000,
			"queue_size": 2000,
			"sample_only": [],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "perf",
				"type": "perf_result",
				"timeout": 300,
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"schema": {
					"$ref": "//../../schema/perf_results.json"
				},
				"debug": true,
				"limit_replicas": false
			},
			"source": {
				"bucket": "active-data-perf",
				"$ref": "file://~/private.json#aws_credentials",
				"key_format": "t.a:b.c"
			}
		},
		{
			"name": "jobs",
			"rollover": {
				"field": "build.date",
				"interval": "3month",
				"max":"2year"
			},
			"batch_size": 500,
			"sample_only": [],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "jobs",
				"type": "job",
				"timeout": 300,
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"schema": {
					"$ref": "//../../schema/jobs.json"
				},
				"debug": false,
				"limit_replicas": false
			},
			"source": {
				"bucket": "active-data-jobs",
				"$ref": "file://~/private.json#aws_credentials",
				"key_format": "a.b"
			}
		},
		{
			"name": "code coverage",
			"skip": 0.0,
			"queue_size": 2000,
			"batch_size": 1000,
			"rollover": {
				"field": "repo.push.date",
				"interval": "day",
				"max": "5day"
			},
			"sample_only": [],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "coverage",
				"type": "code_coverage",
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"timeout": 300,
				"schema": {
					"$ref": "//../../schema/code_coverage.json"
				},
				"debug": false,
				"limit_replicas": false
			},
			"source": {
				"bucket": "active-data-codecoverage",
				"$ref": "file://~/private.json#aws_credentials",
				"key_format": "t.a:b.c.d"
			}
		},
		{
			"name": "task",
			"queue_size": 2000,
			"batch_size": 1000,
			"rollover": {
				"field": "task.created",
				"interval": "3month",
				"max": "2year"
			},
			"sample_only": [],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "task",
				"type": "task",
				"timeout": 300,
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"schema": {
					"$ref": "//../../schema/task_cluster.json"
				},
				"debug": false,
				"limit_replicas": false
			},
			"source": {
				"bucket": "active-data-task-cluster-normalized",
				"$ref": "file://~/private.json#aws_credentials",
				"key_format": "t.a:b"
			}
		}
	],
	"constants": {
		"pyLibrary.env.http.default_headers": {
			"Referer": "https://wiki.mozilla.org/Auto-tools/Projects/ActiveData",
			"User-Agent": "ActiveData-ETL"
		}
	},
	"debug": {
		"trace": true,
		"log": [
			{
				"log_type": "console"
			},
			{
				"class" : "logging.handlers.RotatingFileHandler",
				"filename" : "logs/push_to_es_direct.log",
				"maxBytes" : 10000000,
				"backupCount" : 10,
				"encoding" : "utf8"
			},
			{
				"log_type": "ses",
				"from_address": "<EMAIL>",
				"to_address": "<EMAIL>",
				"subject": "[ALERT][Staging]Problem in Push to ES",
				"$ref": "file://~/private.json#aws_credentials"
			}
		]
	}
}
<commit_msg>less skipping<commit_after>{
	"work_queue": {
		"name": "active-data-index",
		"debug": true,
		"$ref": "file://~/private.json#aws_credentials"
	},
	"workers": [
		{
			"name": "unit",
			"batch_size": 1000,
			"skip": 0.50,
			"rollover":{
				"field": "build.date",
				"interval": "week",
				"max": "5week"
			},
			"sample_only": [
			],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "unittest",
				"type": "test_result",
				"timeout": 300,
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"schema": {
					"$ref": "//../../schema/test_results.json"
				},
				"debug": false,
				"limit_replicas": false
			},
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "active-data-test-result",
				"key_format": "t.a:b.c"
			}
		},
		{
			"name": "perf",
			"rollover":{
				"field": "build.date",
				"interval": "year",
				"max":"2year"
			},
			"batch_size": 1000,
			"queue_size": 2000,
			"sample_only": [],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "perf",
				"type": "perf_result",
				"timeout": 300,
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"schema": {
					"$ref": "//../../schema/perf_results.json"
				},
				"debug": true,
				"limit_replicas": false
			},
			"source": {
				"bucket": "active-data-perf",
				"$ref": "file://~/private.json#aws_credentials",
				"key_format": "t.a:b.c"
			}
		},
		{
			"name": "jobs",
			"rollover": {
				"field": "build.date",
				"interval": "3month",
				"max":"2year"
			},
			"batch_size": 500,
			"sample_only": [],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "jobs",
				"type": "job",
				"timeout": 300,
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"schema": {
					"$ref": "//../../schema/jobs.json"
				},
				"debug": false,
				"limit_replicas": false
			},
			"source": {
				"bucket": "active-data-jobs",
				"$ref": "file://~/private.json#aws_credentials",
				"key_format": "a.b"
			}
		},
		{
			"name": "code coverage",
			"skip": 0.0,
			"queue_size": 2000,
			"batch_size": 1000,
			"rollover": {
				"field": "repo.push.date",
				"interval": "day",
				"max": "5day"
			},
			"sample_only": [],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "coverage",
				"type": "code_coverage",
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"timeout": 300,
				"schema": {
					"$ref": "//../../schema/code_coverage.json"
				},
				"debug": false,
				"limit_replicas": false
			},
			"source": {
				"bucket": "active-data-codecoverage",
				"$ref": "file://~/private.json#aws_credentials",
				"key_format": "<KEY>"
			}
		},
		{
			"name": "task",
			"queue_size": 2000,
			"batch_size": 1000,
			"rollover": {
				"field": "task.created",
				"interval": "3month",
				"max": "2year"
			},
			"sample_only": [],
			"elasticsearch": {
				"host": "http://localhost",
				"port": 9200,
				"index": "task",
				"type": "task",
				"timeout": 300,
				"consistency": "one",   // WE ONLY NEED ONE SHARD TO BE AVAILABLE
				"schema": {
					"$ref": "//../../schema/task_cluster.json"
				},
				"debug": false,
				"limit_replicas": false
			},
			"source": {
				"bucket": "active-data-task-cluster-normalized",
				"$ref": "file://~/private.json#aws_credentials",
				"key_format": "t.a:b"
			}
		}
	],
	"constants": {
		"pyLibrary.env.http.default_headers": {
			"Referer": "https://wiki.mozilla.org/Auto-tools/Projects/ActiveData",
			"User-Agent": "ActiveData-ETL"
		}
	},
	"debug": {
		"trace": true,
		"log": [
			{
				"log_type": "console"
			},
			{
				"class" : "logging.handlers.RotatingFileHandler",
				"filename" : "logs/push_to_es_direct.log",
				"maxBytes" : 10000000,
				"backupCount" : 10,
				"encoding" : "utf8"
			},
			{
				"log_type": "ses",
				"from_address": "<EMAIL>",
				"to_address": "<EMAIL>",
				"subject": "[ALERT][Staging]Problem in Push to ES",
				"$ref": "file://~/private.json#aws_credentials"
			}
		]
	}
}

{'fraction_non_alphanumeric': 0.23054545454545455, 'fraction_numerical': 0.021575757575757575, 'mean_word_length': 2.623627580149319, 'pattern_counts': {'":': 266, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2878362', 'n_tokens_mistral': 4221, 'n_tokens_neox': 3165, 'n_words': 579}