<commit_before>/*
 * Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright (C) 2003 EBI, GRL
 * 
 * This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation,
 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package org.ensembl.healthcheck.testcase.variation;

import java.sql.Connection;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

import org.ensembl.healthcheck.DatabaseRegistryEntry;
import org.ensembl.healthcheck.ReportManager;
import org.ensembl.healthcheck.Team;
import org.ensembl.healthcheck.testcase.generic.ComparePreviousVersionBase;
import org.ensembl.healthcheck.util.DBUtils;


/**
 * Compare the number of transcript_variation alleles having each consequence type between the current database and the database on
 * the secondary server.
 */

public class ComparePreviousVersionConsequenceType extends ComparePreviousVersionBase {

	/**
	 * Create a new testcase.
	 */
	public ComparePreviousVersionConsequenceType() {

		addToGroup("variation-release");
		
		setDescription("Compare the number of transcript_variations having each consequence type status in the current database with those from the equivalent database on the secondary server");
		setTeamResponsible(Team.VARIATION);
	}

	/**
	 * Store the SQL queries in a Properties object.
	 */
	private Properties getSQLQueries() {

		// Store all the needed SQL statements in a Properties object
		Properties sqlQueries = new Properties();
		String query;

		// Query getting the structure of the consequence_types column
		query = "DESCRIBE transcript_variation consequence_types";
		sqlQueries.setProperty("describeConsequenceTypes", query);

		// Query counting the number of transcript variations with a particular consequence type
		query = "SELECT SET_ELEMENT, COUNT(*) FROM transcript_variation tv WHERE FIND_IN_SET(SET_ELEMENT,tv.consequence_types)";
		sqlQueries.setProperty("countVariationsByConsequenceType", query);

		return sqlQueries; 
	}

	protected Map getCounts(DatabaseRegistryEntry dbre) {

		Map<String, Integer> counts = new HashMap<String, Integer>();
		Connection con = dbre.getConnection();

		try {
			// Get all the needed SQL statements in a Properties object
			Properties sqlQueries = getSQLQueries();
	
			// First, get the structure of the consequence_types
			String[] description = DBUtils.getRowValues(dbre.getConnection(), sqlQueries.getProperty("describeConsequenceTypes"));
	
			// The second column contains the type, strip out the individual, comma-separated set elements
			String[] setElements = description[1].split(",");
	
			// Loop over the set elements and count the number for each
			for (int i = 0; i < setElements.length; i++) {
	
				// We need to strip away any 'set(' or ')' strings from the set element
				setElements[i] = setElements[i].replaceAll("set\\(|\\)", "");
	
				// Replace the 'SET_ELEMENT' placeholder with the current consequence_type and do the query
				counts.putAll(getCountsBySQL(dbre, sqlQueries.getProperty("countVariationsByConsequenceType").replaceAll("SET_ELEMENT", setElements[i])));
			}
		} catch (Exception e) {
			ReportManager.problem(this, con, "HealthCheck caused an exception: " + e.getMessage());
		}
		
		return counts;
	}

	// ------------------------------------------------------------------------

	protected String entityDescription() {

		return "consequence type";

	}

	// ------------------------------------------------------------------------

	protected double threshold() {

		return 0.95;
<commit_msg>change counting to speed up<commit_after>/*
 * Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright (C) 2003 EBI, GRL
 * 
 * This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation,
 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package org.ensembl.healthcheck.testcase.variation;

import java.sql.Connection;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

import org.ensembl.healthcheck.DatabaseRegistryEntry;
import org.ensembl.healthcheck.ReportManager;
import org.ensembl.healthcheck.Team;
import org.ensembl.healthcheck.testcase.generic.ComparePreviousVersionBase;
import org.ensembl.healthcheck.util.DBUtils;


/**
 * Compare the number of transcript_variation alleles having each consequence type between the current database and the database on
 * the secondary server.
 */

public class ComparePreviousVersionConsequenceType extends ComparePreviousVersionBase {

	/**
	 * Create a new testcase.
	 */
	public ComparePreviousVersionConsequenceType() {

		addToGroup("variation-release");
		
		setDescription("Compare the number of transcript_variations having each consequence type status in the current database with those from the equivalent database on the secondary server");
		setTeamResponsible(Team.VARIATION);
	}


	protected Map getCounts(DatabaseRegistryEntry dbre) {

		Map<String, Integer> counts = new HashMap<String, Integer>();
		Connection con = dbre.getConnection();

		try {
			// 
			counts.putAll(getCountsBySQL(dbre, "SELECT consequence_types, COUNT(*) FROM transcript_variation group by consequence_types")); 
			
		} catch (Exception e) {
			ReportManager.problem(this, con, "HealthCheck caused an exception: " + e.getMessage());
		}
		
		return counts;
	}

	// ------------------------------------------------------------------------

	protected String entityDescription() {

		return "consequence type";

	}

	// ------------------------------------------------------------------------

	protected double threshold() {

		return 0.95;

{'fraction_non_alphanumeric': 0.11198600174978128, 'fraction_numerical': 0.008998875140607425, 'mean_word_length': 3.630787037037037, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5836159', 'n_tokens_mistral': 2197, 'n_tokens_neox': 1976, 'n_words': 819}