<commit_before><dataConfig>

	<script>
	<![CDATA[
		function getData(row) {
			if (row.get('observation_type') == 'categorical') {
				row.put('category', row.get('category'));

			} else if (row.get('observation_type') == 'unidimensional') {
				row.put('data_point', row.get('unidimensional_data_point'));

			} else if (row.get('observation_type') == 'multidimensional') {
				row.put('data_point', row.get('multidimensional_data_point'));
				row.put('order_index', row.get('order_index'));
				row.put('dimension', row.get('dimension'));

			} else if (row.get('observation_type') == 'time_series') {
				row.put('data_point', row.get('time_series_data_point'));
				row.put('time_point', row.get('time_point'));
				row.put('discrete_point', row.get('discrete_point'));

			} else if (row.get('observation_type') == 'metadata') {
				row.put('value', row.get('value'));

			} else if (row.get('observation_type') == 'image_record') {
				row.put('file_type', row.get('file_type'));
				row.put('full_resolution_file_path', row.get('full_resolution_file_path'));
				row.put('small_thumbnail_file_path', row.get('small_thumbnail_file_path'));
			}

			return row;
		}

	]]>
	</script>

	<dataSource name="komp2ds" jndiName="java:comp/env/jdbc/komp2DataSource"
		batchSize="-1" readOnly="true" />


	<document name="komp2_indexing">

		<!-- For all observations (data points) -->
		<entity dataSource="komp2ds" transformer="script:getData" name="observation"
			query="
			SELECT strain.strain_acc, pipe.name as pipeline_name, pipe.id as pipeline_id, pipe.stable_id as pipeline_stable_id, param.name as parameter_name, proc.name as procedure_name, proc.id as procedure_id, proc.stable_id as procedure_stable_id, param.stable_id as parameter_stable_id,bs.organisation_id, org.name, bs.sample_group, gf.acc as acc, ls.sex as sex, ls.colony_id, ls.zygosity, gf.symbol as symbol, bms.biological_model_id as bmid, ls.date_of_birth as birthdate, ls.colony_id, e.date_of_experiment, e.external_id, e.id as experiment_id, e.metadata_combined as metadata_combined, e.metadata_group as metadata_group, o.*, co.category as category, uo.data_point as unidimensional_data_point, mo.data_point as multidimensional_data_point, mo.order_index, mo.dimension, tso.data_point as time_series_data_point, tso.time_point, tso.discrete_point, iro.file_type, iro.full_resolution_file_path, iro.small_thumbnail_file_path, bs.external_id as external_sample_id
			FROM observation o
			LEFT OUTER JOIN categorical_observation co ON o.id=co.id
			LEFT OUTER JOIN unidimensional_observation uo ON o.id=uo.id
			LEFT OUTER JOIN multidimensional_observation mo ON o.id=mo.id
			LEFT OUTER JOIN time_series_observation tso ON o.id=tso.id
			LEFT OUTER JOIN image_record_observation iro ON o.id=iro.id
			INNER JOIN biological_sample bs ON bs.id=o.biological_sample_id
			INNER JOIN organisation org ON bs.organisation_id=org.id
			INNER JOIN live_sample ls ON bs.id=ls.id
			INNER JOIN biological_model_sample bms ON bs.id=bms.biological_sample_id
			INNER JOIN biological_model_strain strain ON strain.biological_model_id=bms.biological_model_id
			LEFT OUTER JOIN biological_model_genomic_feature bmgf ON bmgf.biological_model_id=bms.biological_model_id
			LEFT OUTER JOIN genomic_feature gf ON gf.acc=bmgf.gf_acc<commit_msg>remove metadata comparison in transformer (it will never be true now)<commit_after><dataConfig>

	<script>
	<![CDATA[
		function getData(row) {
			if (row.get('observation_type') == 'categorical') {
				row.put('category', row.get('category'));

			} else if (row.get('observation_type') == 'unidimensional') {
				row.put('data_point', row.get('unidimensional_data_point'));

			} else if (row.get('observation_type') == 'time_series') {
				row.put('data_point', row.get('time_series_data_point'));
				row.put('time_point', row.get('time_point'));
				row.put('discrete_point', row.get('discrete_point'));

			} else if (row.get('observation_type') == 'image_record') {
				row.put('file_type', row.get('file_type'));
				row.put('full_resolution_file_path', row.get('full_resolution_file_path'));
				row.put('small_thumbnail_file_path', row.get('small_thumbnail_file_path'));

			} else if (row.get('observation_type') == 'multidimensional') {
				row.put('data_point', row.get('multidimensional_data_point'));
				row.put('order_index', row.get('order_index'));
				row.put('dimension', row.get('dimension'));

			}

			return row;
		}

	]]>
	</script>

	<dataSource name="komp2ds" jndiName="java:comp/env/jdbc/komp2DataSource"
		batchSize="-1" readOnly="true" />


	<document name="komp2_indexing">

		<!-- For all observations (data points) -->
		<entity dataSource="komp2ds" transformer="script:getData" name="observation"
			query="
			SELECT strain.strain_acc, pipe.name as pipeline_name, pipe.id as pipeline_id, pipe.stable_id as pipeline_stable_id, param.name as parameter_name, proc.name as procedure_name, proc.id as procedure_id, proc.stable_id as procedure_stable_id, param.stable_id as parameter_stable_id,bs.organisation_id, org.name, bs.sample_group, gf.acc as acc, ls.sex as sex, ls.colony_id, ls.zygosity, gf.symbol as symbol, bms.biological_model_id as bmid, ls.date_of_birth as birthdate, ls.colony_id, e.date_of_experiment, e.external_id, e.id as experiment_id, e.metadata_combined as metadata_combined, e.metadata_group as metadata_group, o.*, co.category as category, uo.data_point as unidimensional_data_point, mo.data_point as multidimensional_data_point, mo.order_index, mo.dimension, tso.data_point as time_series_data_point, tso.time_point, tso.discrete_point, iro.file_type, iro.full_resolution_file_path, iro.small_thumbnail_file_path, bs.external_id as external_sample_id
			FROM observation o
			LEFT OUTER JOIN categorical_observation co ON o.id=co.id
			LEFT OUTER JOIN unidimensional_observation uo ON o.id=uo.id
			LEFT OUTER JOIN multidimensional_observation mo ON o.id=mo.id
			LEFT OUTER JOIN time_series_observation tso ON o.id=tso.id
			LEFT OUTER JOIN image_record_observation iro ON o.id=iro.id
			INNER JOIN biological_sample bs ON bs.id=o.biological_sample_id
			INNER JOIN organisation org ON bs.organisation_id=org.id
			INNER JOIN live_sample ls ON bs.id=ls.id
			INNER JOIN biological_model_sample bms ON bs.id=bms.biological_sample_id
			INNER JOIN biological_model_strain strain ON strain.biological_model_id=bms.biological_model_id
			LEFT OUTER JOIN biological_model_genomic_feature bmgf ON bmgf.biological_model_id=bms.biological_model_id
			LEFT OUTER JOIN genomic_feature gf ON gf.acc=bmgf.gf_acc
{'fraction_non_alphanumeric': 0.1194508009153318, 'fraction_numerical': 0.0015255530129672007, 'mean_word_length': 5.556, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1552715', 'n_tokens_mistral': 2521, 'n_tokens_neox': 2221, 'n_words': 471}