<commit_before>{
	"work_queue": {
		"name": "ekyle-etl",
		"debug": true,
		"$ref": "file://~/private.json#aws_credentials"
	},
	"param": {
		"threads": 1,
		"wait_forever": true,
		"keep_unknown_on_queue": true
	},
	"constants": {
		"pyLibrary.env.http.default_headers": {
			"From": "https://wiki.mozilla.org/Auto-tools/Projects/ActiveData"
		},
		"pyLibrary.env.big_data.MAX_STRING_SIZE": 10000000
	},
	"workers": [
		{
			"name": "pulse2es",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-pulse-logger"
			},
			"destination": {
				"use_daily": false,
				"host": "http://5172.16.58.3",
				"port": 9200,
				"index": "pulse",
				"type": "pulse_log",
				"timeout": 300,
				"schema": {
					"$ref": "//../schema/pulse_logs.json"
				},
				"debug": true,
				"limit_replicas": true
			},
			"transformer": "testlog_etl.transforms.pulse_block_to_es.process_pulse_block_to_es",
			"type": "join"
		},
		{
			"name": "pulse2talos",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-pulse-logger"
			},
			"destination": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-talos",
				"public": true
			},
			"transformer": "testlog_etl.transforms.pulse_block_to_talos_logs.process_talos",
			"type": "join"
		},
		{
			"name": "pulse2test_result",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-pulse-logger"
			},
			"destination": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-test-result"
			},
			"transformer": "testlog_etl.transforms.pulse_block_to_test_result_logs.process_talos",
			"type": "join"
		},
		{
			"name":"load es",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-test-result"
			},
			"destination": {
				"use_daily": true,
				"host": "http://5172.16.58.3",
				"port": 9200,
				"index": "unittest",
				"type": "test_result",
				"timeout": 300,
				"schema": {
					"$ref": "//../schema/test_results.json"
				},
				"debug": true,
				"limit_replicas": true
			},
			"transformer": "testlog_etl.transforms.test_result_to_sink.process_test_result",
			"type": "join"

		},
		{
			"name":"load redshift",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-test-result"
			},
			"destination": {
				"type": "redshift",
				"host": "ekyle-redshift-test.cishavsxiep4.us-west-2.redshift.amazonaws.com",
				"port": 5439,
				"database": "unittest",
				"table": "test_results",
				"$ref": "file://~/private.json#redshift",
				"mapping": {
					"$ref": "//../schema/redshift.json"
				},
				"timeout": 300,
				"debug": true,
				"source":{
					"$ref":"#...source"
				},
				"meta": {
					"bucket": "ekyle-metadata",
					"jsonspath": "push_to_redshift",
					"$ref": "file://~/private.json#aws_credentials"
				}
			},<commit_msg>maybe es is causing stall<commit_after>{
	"work_queue": {
		"name": "ekyle-etl",
		"debug": true,
		"$ref": "file://~/private.json#aws_credentials"
	},
	"param": {
		"threads": 1,
		"wait_forever": true,
		"keep_unknown_on_queue": true
	},
	"constants": {
		"pyLibrary.env.http.default_headers": {
			"From": "https://wiki.mozilla.org/Auto-tools/Projects/ActiveData"
		},
		"pyLibrary.env.big_data.MAX_STRING_SIZE": 10000000
	},
	"workers": [
		{
			"name": "pulse2es",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-pulse-logger"
			},
			"destination": {
				"use_daily": false,
				"host": "http://172.16.31.10",
				"port": 9200,
				"index": "pulse",
				"type": "pulse_log",
				"timeout": 300,
				"schema": {
					"$ref": "//../schema/pulse_logs.json"
				},
				"debug": true,
				"limit_replicas": true
			},
			"transformer": "testlog_etl.transforms.pulse_block_to_es.process_pulse_block_to_es",
			"type": "join"
		},
		{
			"name": "pulse2talos",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-pulse-logger"
			},
			"destination": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-talos",
				"public": true
			},
			"transformer": "testlog_etl.transforms.pulse_block_to_talos_logs.process_talos",
			"type": "join"
		},
		{
			"name": "pulse2test_result",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-pulse-logger"
			},
			"destination": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-test-result"
			},
			"transformer": "testlog_etl.transforms.pulse_block_to_test_result_logs.process_talos",
			"type": "join"
		},
//		{
//			"name":"load es",
//			"source": {
//				"$ref": "file://~/private.json#aws_credentials",
//				"bucket": "ekyle-test-result"
//			},
//			"destination": {
//				"use_daily": true,
//				"host": "http://52.10.189.133",
//				"port": 9200,
//				"index": "unittest",
//				"type": "test_result",
//				"timeout": 300,
//				"schema": {
//					"$ref": "//../schema/test_results.json"
//				},
//				"debug": true,
//				"limit_replicas": true
//			},
//			"transformer": "testlog_etl.transforms.test_result_to_sink.process_test_result",
//			"type": "join"
//
//		},
		{
			"name":"load redshift",
			"source": {
				"$ref": "file://~/private.json#aws_credentials",
				"bucket": "ekyle-test-result"
			},
			"destination": {
				"type": "redshift",
				"host": "ekyle-redshift-test.cishavsxiep4.us-west-2.redshift.amazonaws.com",
				"port": 5439,
				"database": "unittest",
				"table": "test_results",
				"$ref": "file://~/private.json#redshift",
				"mapping": {
					"$ref": "//../schema/redshift.json"
				},
				"timeout": 300,
				"debug": true,
				"source":{
					"$ref":"#...source"
				},
				"meta": {
					"bucket": "ekyle-metadata",
					"jsonspath": "push_to_redshift",
					"$ref": "file://~/private.json#aws_credentials"
				}
			},
{'fraction_non_alphanumeric': 0.22765055131467346, 'fraction_numerical': 0.0181509754028838, 'mean_word_length': 3.0578114246386785, 'pattern_counts': {'":': 174, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1556687', 'n_tokens_mistral': 2946, 'n_tokens_neox': 2318, 'n_words': 312}