<commit_before>/** Copyright (c) 2012 Memorial Sloan-Kettering Cancer Center.
**
** This library is free software; you can redistribute it and/or modify it
** under the terms of the GNU Lesser General Public License as published
** by the Free Software Foundation; either version 2.1 of the License, or
** any later version.
**
** This library is distributed in the hope that it will be useful, but
** WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF
** MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.  The software and
** documentation provided hereunder is on an "as is" basis, and
** Memorial Sloan-Kettering Cancer Center 
** has no obligations to provide maintenance, support,
** updates, enhancements or modifications.  In no event shall
** Memorial Sloan-Kettering Cancer Center
** be liable to any party for direct, indirect, special,
** incidental or consequential damages, including lost profits, arising
** out of the use of this software and its documentation, even if
** Memorial Sloan-Kettering Cancer Center 
** has been advised of the possibility of such damage.  See
** the GNU Lesser General Public License for more details.
**
** You should have received a copy of the GNU Lesser General Public License
** along with this library; if not, write to the Free Software Foundation,
** Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
**/

// package
package org.mskcc.cbio.importer;

// imports
import org.mskcc.cbio.importer.Config;
import org.mskcc.cbio.importer.Fetcher;
import org.mskcc.cbio.importer.DatabaseUtils;
import org.mskcc.cbio.importer.model.PortalMetadata;
import org.mskcc.cbio.importer.model.DatatypeMetadata;
import org.mskcc.cbio.importer.model.DataSourcesMetadata;
import org.mskcc.cbio.importer.model.ReferenceMetadata;

import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.CommandLineParser;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.PropertyConfigurator;

import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import java.io.File;
import java.io.PrintWriter;
import java.util.Map;
import java.util.HashSet;
import java.util.Collection;
import java.util.Properties;
import java.text.SimpleDateFormat;

/**
 * Class which provides command line admin capabilities 
 * to the importer tool.
 */
public class Admin implements Runnable {

	// our context file
	public static final String contextFile = "classpath:applicationContext-importer.xml";

	// date format 
	public static final SimpleDateFormat PORTAL_DATE_FORMAT = new SimpleDateFormat("MM/dd/yyyy");

	// context
	private static final ApplicationContext context = new ClassPathXmlApplicationContext(contextFile);

	// our logger
	private static final Log LOG = LogFactory.getLog(Admin.class);

	// options var
	private static final Options options = initializeOptions();

	// identifiers for init db command
	private static final String PORTAL_DATABASE = "portal";
	private static final String IMPORTER_DATABASE = "importer";

	// parsed command line
	private CommandLine commandLine;

	/**
	 * Method to get beans by id
	 *
	 * @param String beanID
	 * @return Object
	 */
	private static Object getBean(String beanID) {
		return context.getBean(beanID);
	}

	/**
	 * Method to initialize our static options var
	 *
	 * @return Options
	 */
	private static Options initializeOptions() {
		
		// create each option
		Option help = new Option("help", "Print this message.");

        Option initializeDatabase = (OptionBuilder.withArgName("db_name")
									 .hasArg()
									 .withDescription("Initialize database(s).  Valid " +
													  "database identifiers are: " +
													  "\"" + PORTAL_DATABASE + "\" and \"" +
													  IMPORTER_DATABASE + "\" or " +
													  "\"" + Config.ALL + "\".")
									 .create("init_db"));

        Option fetchData = (OptionBuilder.withArgName("data_source:run_date")
							.hasArgs(2)
							.withValueSeparator(':')
							.withDescription("Fetch data from the given data_source and the given run date (mm/dd/yyyy).  " + 
											 "Use \"" + Fetcher.LATEST_RUN_INDICATOR + "\" to retrieve the most current run.")
							.create("fetch_data"));

        Option fetchReferenceData = (OptionBuilder.withArgName("reference_data")
									  .hasArg()
									  .withDescription("Fetch the given reference data." +
													   "  Use \"" + Config.ALL + "\" to retrieve all reference data.")
									  .create("fetch_reference_data"));

        Option oncotateMAF = (OptionBuilder.withArgName("maf_file")
							  .hasArg()
							  .withDescription("Run the given MAF though the Oncotator and OMA tools.")
							  .create("oncotate_maf"));

        Option oncotateAllMAFs = (OptionBuilder.withArgName("data_source")
							  .hasArg()
							  .withDescription("Run all MAFs in the given datasource though the Oncotator and OMA tools.")
							  .create("oncotate_mafs"));

        Option convertData = (OptionBuilder.withArgName("portal:run_date:apply_overrides")
                              .hasArgs(3)
							  .withValueSeparator(':')
                              .withDescription("Convert data within the importer database " +
											   "from the given run date (mm/dd/yyyy), " +
											   "for the given portal.  If apply_overrides is 't', " +
											   "overrides will be substituted for data_source data " +
											   "before staging files are created.")
                              .create("convert_data"));

		Option applyOverrides = (OptionBuilder.withArgName("portal:exclude_datatype")		
								 .hasArgs(2)		
								 .withDescription("Replace staging files for the given portal " +
												  "with any exisiting overrides.  If exclude_datatype is set, " +
												  "the datatype provided will not have overrides applied.")
								 .create("apply_overrides"));
<commit_msg>applyOverrides has argument to skip applying an override for a given datatype<commit_after>/** Copyright (c) 2012 Memorial Sloan-Kettering Cancer Center.
**
** This library is free software; you can redistribute it and/or modify it
** under the terms of the GNU Lesser General Public License as published
** by the Free Software Foundation; either version 2.1 of the License, or
** any later version.
**
** This library is distributed in the hope that it will be useful, but
** WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF
** MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.  The software and
** documentation provided hereunder is on an "as is" basis, and
** Memorial Sloan-Kettering Cancer Center 
** has no obligations to provide maintenance, support,
** updates, enhancements or modifications.  In no event shall
** Memorial Sloan-Kettering Cancer Center
** be liable to any party for direct, indirect, special,
** incidental or consequential damages, including lost profits, arising
** out of the use of this software and its documentation, even if
** Memorial Sloan-Kettering Cancer Center 
** has been advised of the possibility of such damage.  See
** the GNU Lesser General Public License for more details.
**
** You should have received a copy of the GNU Lesser General Public License
** along with this library; if not, write to the Free Software Foundation,
** Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
**/

// package
package org.mskcc.cbio.importer;

// imports
import org.mskcc.cbio.importer.Config;
import org.mskcc.cbio.importer.Fetcher;
import org.mskcc.cbio.importer.DatabaseUtils;
import org.mskcc.cbio.importer.model.PortalMetadata;
import org.mskcc.cbio.importer.model.DatatypeMetadata;
import org.mskcc.cbio.importer.model.DataSourcesMetadata;
import org.mskcc.cbio.importer.model.ReferenceMetadata;

import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.CommandLineParser;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.PropertyConfigurator;

import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import java.io.File;
import java.io.PrintWriter;
import java.util.Map;
import java.util.HashSet;
import java.util.Collection;
import java.util.Properties;
import java.text.SimpleDateFormat;

/**
 * Class which provides command line admin capabilities 
 * to the importer tool.
 */
public class Admin implements Runnable {

	// our context file
	public static final String contextFile = "classpath:applicationContext-importer.xml";

	// date format 
	public static final SimpleDateFormat PORTAL_DATE_FORMAT = new SimpleDateFormat("MM/dd/yyyy");

	// context
	private static final ApplicationContext context = new ClassPathXmlApplicationContext(contextFile);

	// our logger
	private static final Log LOG = LogFactory.getLog(Admin.class);

	// options var
	private static final Options options = initializeOptions();

	// identifiers for init db command
	private static final String PORTAL_DATABASE = "portal";
	private static final String IMPORTER_DATABASE = "importer";

	// parsed command line
	private CommandLine commandLine;

	/**
	 * Method to get beans by id
	 *
	 * @param String beanID
	 * @return Object
	 */
	private static Object getBean(String beanID) {
		return context.getBean(beanID);
	}

	/**
	 * Method to initialize our static options var
	 *
	 * @return Options
	 */
	private static Options initializeOptions() {
		
		// create each option
		Option help = new Option("help", "Print this message.");

        Option initializeDatabase = (OptionBuilder.withArgName("db_name")
									 .hasArg()
									 .withDescription("Initialize database(s).  Valid " +
													  "database identifiers are: " +
													  "\"" + PORTAL_DATABASE + "\" and \"" +
													  IMPORTER_DATABASE + "\" or " +
													  "\"" + Config.ALL + "\".")
									 .create("init_db"));

        Option fetchData = (OptionBuilder.withArgName("data_source:run_date")
							.hasArgs(2)
							.withValueSeparator(':')
							.withDescription("Fetch data from the given data_source and the given run date (mm/dd/yyyy).  " + 
											 "Use \"" + Fetcher.LATEST_RUN_INDICATOR + "\" to retrieve the most current run.")
							.create("fetch_data"));

        Option fetchReferenceData = (OptionBuilder.withArgName("reference_data")
									  .hasArg()
									  .withDescription("Fetch the given reference data." +
													   "  Use \"" + Config.ALL + "\" to retrieve all reference data.")
									  .create("fetch_reference_data"));

        Option oncotateMAF = (OptionBuilder.withArgName("maf_file")
							  .hasArg()
							  .withDescription("Run the given MAF though the Oncotator and OMA tools.")
							  .create("oncotate_maf"));

        Option oncotateAllMAFs = (OptionBuilder.withArgName("data_source")
							  .hasArg()
							  .withDescription("Run all MAFs in the given datasource though the Oncotator and OMA tools.")
							  .create("oncotate_mafs"));

        Option convertData = (OptionBuilder.withArgName("portal:run_date:apply_overrides")
                              .hasArgs(3)
							  .withValueSeparator(':')
                              .withDescription("Convert data within the importer database " +
											   "from the given run date (mm/dd/yyyy), " +
											   "for the given portal.  If apply_overrides is 't', " +
											   "overrides will be substituted for data_source data " +
											   "before staging files are created.")
                              .create("convert_data"));

		Option applyOverrides = (OptionBuilder.withArgName("portal:exclude_datatype")		
								 .hasArgs(2)		
								 .withValueSeparator(':')
								 .withDescription("Replace staging files for the given portal " +
												  "with any exisiting overrides.  If exclude_datatype is set, " +
												  "the datatype provided will not have overrides applied.")
								 .create("apply_overrides"));

{'fraction_non_alphanumeric': 0.0970795892169448, 'fraction_numerical': 0.0038510911424903724, 'mean_word_length': 3.308676114759765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2366275', 'n_tokens_mistral': 4051, 'n_tokens_neox': 3336, 'n_words': 1082}