<commit_before>#!/bin/bash
# kibala-import-bacula-pool.sh
# Dump pool definitions from Bacula database and import into ElasticSearch for kibala visualization

# Load configuration
source $(dirname $0)/kibala.conf

# Generate and insert documents for elasticsearch of kibala
mysql	--silent --raw \
	-h$BACULA_DB_HOST \
	-u$BACULA_DB_USERNAME \
	$BACULA_DB_SCHEMA >/tmp/kibala-spool <<EOF
select concat(
	'{ "index": { "_index": "$ES_INDEX", "_type": "Pool", "_id": ', p.PoolId, ' } }\n',
	'{ "@timestamp": "', 		date_format(now(), '%Y-%m-%dT%H:%i:%s'), '"',
	', "PoolId": ',			p.PoolId,
	', "PoolName": "',		p.Name, '"',
	', "PoolType": "',		p.PoolType, '"',
	', "VolRetention": ',		p.VolRetention,
	', "VolUseDuration": ',		p.VolUseDuration,
	', "AutoPrune": ',		p.AutoPrune,
	', "Recycle": ',		p.Recycle,
	', "Enabled": ',		p.Enabled,
	', "ClientName": "',		ifnull( ( select group_concat(	DISTINCT c.Name
									order by c.Name separator ' ')
						  from Job j
						  left join Client c on j.ClientId = c.ClientId
						  where j.PoolId = p.PoolId
						),
						''
					), '"',
	', "JobName": "',		ifnull( ( select group_concat( 	DISTINCT j.Name
									order by j.Name separator ' ')
						  from Job j
						  where j.PoolId = p.PoolId
						),
						''
					), '"',
	' }'
) output
from 	Pool p
EOF

curl -s -XPOST $ES_URL/_bulk --data-binary @/tmp/kibala-spool && rm /tmp/kibala-spool

echo<commit_msg>Reference jobs and clients for pool type as list<commit_after>#!/bin/bash
# kibala-import-bacula-pool.sh
# Dump pool definitions from Bacula database and import into ElasticSearch for kibala visualization

# Load configuration
source $(dirname $0)/kibala.conf

# Generate and insert documents for elasticsearch of kibala
mysql	--silent --raw \
	-h$BACULA_DB_HOST \
	-u$BACULA_DB_USERNAME \
	$BACULA_DB_SCHEMA >/tmp/kibala-spool <<EOF
select concat(
	'{ "index": { "_index": "$ES_INDEX", "_type": "Pool", "_id": ', p.PoolId, ' } }\n',
	'{ "@timestamp": "', 		date_format(now(), '%Y-%m-%dT%H:%i:%s'), '"',
	', "PoolId": ',			p.PoolId,
	', "PoolName": "',		p.Name, '"',
	', "PoolType": "',		p.PoolType, '"',
	', "VolRetention": ',		p.VolRetention,
	', "VolUseDuration": ',		p.VolUseDuration,
	', "AutoPrune": ',		p.AutoPrune,
	', "Recycle": ',		p.Recycle,
	', "Enabled": ',		p.Enabled,
	', "ClientName": [ ',		ifnull( ( select group_concat(	DISTINCT concat('"', c.Name, '"')
									order by c.Name separator ', ')
						  from Job j
						  left join Client c on j.ClientId = c.ClientId
						  where j.PoolId = p.PoolId
						),
						''
					), ' ]',
	', "JobName": [ ',		ifnull( ( select group_concat( 	DISTINCT concat('"', j.Name, '"')
									order by j.Name separator ', ')
						  from Job j
						  where j.PoolId = p.PoolId
						),
						''
					), ' ]',
	' }'
) output
from 	Pool p
EOF

curl -s -XPOST $ES_URL/_bulk --data-binary @/tmp/kibala-spool && rm /tmp/kibala-spool

echo
{'fraction_non_alphanumeric': 0.1909686315063771, 'fraction_numerical': 0.0006894174422612892, 'mean_word_length': 2.98079561042524, 'pattern_counts': {'":': 30, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1754856', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1130, 'n_words': 245}