<commit_before>package org.biojava.bio.structure.align;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.biojava.bio.structure.align.ce.AbstractUserArgumentProcessor;
import org.biojava.bio.structure.align.client.FarmJobParameters;
import org.biojava.bio.structure.align.client.FarmJobRunnable;
import org.biojava.bio.structure.align.events.AlignmentProgressListener;
import org.biojava.bio.structure.align.util.CliTools;
import org.biojava.bio.structure.align.util.ConfigurationException;
import org.biojava.bio.structure.scop.CachedRemoteScopInstallation;
import org.biojava.bio.structure.scop.ScopDatabase;
import org.biojava.bio.structure.scop.ScopFactory;
import org.biojava3.core.util.InputStreamProvider;


/** A job as it can be run on the farm.
 * 
 * @author <NAME>
 * 
 * for arguments see the printHelp() method.
 * 
 * 
 *
 */
public class FarmJob implements Runnable {

	private static final String[] mandParams = new String[] {"pdbFilePath"};

	private static final List<String> mandatoryArgs= Arrays.asList(mandParams);

	List<AlignmentProgressListener> progressListeners;
	List<FarmJobRunnable> jobs;

	FarmJobParameters params ;
	
	public FarmJob(){
		progressListeners = null;
		
		// send a flag to the PDb file loader to cache the gzip compressed files.
		System.setProperty(InputStreamProvider.CACHE_PROPERTY, "true");

		
	}
	
	public FarmJobParameters getParams() {
		return params;
	}

	public void setParams(FarmJobParameters params) {
		this.params = params;
	}

	public void addAlignmentProgressListener(AlignmentProgressListener listener){
		if (progressListeners == null)
			progressListeners = new ArrayList<AlignmentProgressListener>();
		
		progressListeners.add(listener);
	}
	
	public void clearListeners(){
		progressListeners.clear();
		progressListeners = null;
	}

	public static void main(String[] argv){

		FarmJob job = new FarmJob();

		if (argv.length  == 0 ) {			
			job.printHelp();
			return;			
		}

		if ( argv.length == 1){
			if (argv[0].equalsIgnoreCase("-h") || argv[0].equalsIgnoreCase("-help")|| argv[0].equalsIgnoreCase("--help")){
				job.printHelp();								
				return;
			}
		}

		FarmJobParameters params = new FarmJobParameters();
		
		for (int i = 0 ; i < argv.length; i++){
			String arg   = argv[i];

			String value = null;
			if ( i < argv.length -1)
				value = argv[i+1];

			// if value starts with - then the arg does not have a value.
			if (value != null && value.startsWith("-"))
				value = null;
			else 
				i++;


			String[] tmp = {arg,value};

			try {

				CliTools.configureBean(params, tmp);  

			} catch (ConfigurationException e){

				e.printStackTrace();

				if ( mandatoryArgs.contains(arg) ) {
					// there must not be a ConfigurationException with mandatory arguments.
					return;
				} else {
					// but there can be with optional ...
				}
			}    
		}
				
		
		if (( params.getNrAlignments() == -1) && (params.getTime() == -1)){
			System.err.println("Please provide either the -time or the -nrAlignments argument!");
			return;
		}
		
		

		
		FarmJobRunnable.log("Using parameters: " + params.toString());
		
		job.setParams(params);
		job.run();
		
	}
	
	public void run(){

		
		// set the system wide PDB path

		String path = params.getPdbFilePath();
		System.setProperty(AbstractUserArgumentProcessor.PDB_DIR,path);
				
		// declare SCOP to be locally cached, but fetching new stuff from remote
		ScopDatabase scop = new CachedRemoteScopInstallation(true);
		ScopFactory.setScopDatabase(scop);
		
		String username = params.getUsername();
		jobs = new ArrayList<FarmJobRunnable>();
		for ( int i = 0 ; i < params.getThreads();i++){
			FarmJobRunnable.log("starting thread #" + (i+1));
			FarmJobRunnable runner = new FarmJobRunnable(params);	
			params.setUsername(username+"_thread_" + (i+1));
			jobs.add(runner);
			
			if ( progressListeners != null) {
				for (AlignmentProgressListener li : progressListeners){
					runner.addAlignmentProgressListener(li);
				}<commit_msg>adding support for new parameter cacheFilePath to (optionally) allow to separate PDB files from other data<commit_after>package org.biojava.bio.structure.align;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.biojava.bio.structure.align.ce.AbstractUserArgumentProcessor;
import org.biojava.bio.structure.align.client.FarmJobParameters;
import org.biojava.bio.structure.align.client.FarmJobRunnable;
import org.biojava.bio.structure.align.events.AlignmentProgressListener;
import org.biojava.bio.structure.align.util.CliTools;
import org.biojava.bio.structure.align.util.ConfigurationException;
import org.biojava.bio.structure.scop.CachedRemoteScopInstallation;
import org.biojava.bio.structure.scop.ScopDatabase;
import org.biojava.bio.structure.scop.ScopFactory;
import org.biojava3.core.util.InputStreamProvider;


/** A job as it can be run on the farm.
 * 
 * @author <NAME>
 * 
 * for arguments see the printHelp() method.
 * 
 * 
 *
 */
public class FarmJob implements Runnable {

	private static final String[] mandParams = new String[] {"pdbFilePath"};

	private static final List<String> mandatoryArgs= Arrays.asList(mandParams);

	List<AlignmentProgressListener> progressListeners;
	List<FarmJobRunnable> jobs;

	FarmJobParameters params ;
	
	public FarmJob(){
		progressListeners = null;
		
		// send a flag to the PDb file loader to cache the gzip compressed files.
		System.setProperty(InputStreamProvider.CACHE_PROPERTY, "true");

		
	}
	
	public FarmJobParameters getParams() {
		return params;
	}

	public void setParams(FarmJobParameters params) {
		this.params = params;
	}

	public void addAlignmentProgressListener(AlignmentProgressListener listener){
		if (progressListeners == null)
			progressListeners = new ArrayList<AlignmentProgressListener>();
		
		progressListeners.add(listener);
	}
	
	public void clearListeners(){
		progressListeners.clear();
		progressListeners = null;
	}

	public static void main(String[] argv){

		FarmJob job = new FarmJob();

		if (argv.length  == 0 ) {			
			job.printHelp();
			return;			
		}

		if ( argv.length == 1){
			if (argv[0].equalsIgnoreCase("-h") || argv[0].equalsIgnoreCase("-help")|| argv[0].equalsIgnoreCase("--help")){
				job.printHelp();								
				return;
			}
		}

		FarmJobParameters params = new FarmJobParameters();
		
		for (int i = 0 ; i < argv.length; i++){
			String arg   = argv[i];

			String value = null;
			if ( i < argv.length -1)
				value = argv[i+1];

			// if value starts with - then the arg does not have a value.
			if (value != null && value.startsWith("-"))
				value = null;
			else 
				i++;


			String[] tmp = {arg,value};

			try {

				CliTools.configureBean(params, tmp);  

			} catch (ConfigurationException e){

				e.printStackTrace();

				if ( mandatoryArgs.contains(arg) ) {
					// there must not be a ConfigurationException with mandatory arguments.
					return;
				} else {
					// but there can be with optional ...
				}
			}    
		}
				
		
		if (( params.getNrAlignments() == -1) && (params.getTime() == -1)){
			System.err.println("Please provide either the -time or the -nrAlignments argument!");
			return;
		}
		
		

		
		FarmJobRunnable.log("Using parameters: " + params.toString());
		
		job.setParams(params);
		job.run();
		
	}
	
	public void run(){

		
		// set the system wide PDB path

		String path = params.getPdbFilePath();
		System.setProperty(AbstractUserArgumentProcessor.PDB_DIR,path);
				
		String cachePath = params.getCacheFilePath();
		if ( cachePath != null && ! cachePath.equals(""))
			System.setProperty(AbstractUserArgumentProcessor.CACHE_DIR,cachePath);
		else {
			// if not provided, we use pdbFilePath as the default CACHE path
			System.setProperty(AbstractUserArgumentProcessor.CACHE_DIR,path);
		}
		// declare SCOP to be locally cached, but fetching new stuff from remote
		ScopDatabase scop = new CachedRemoteScopInstallation(true);
		ScopFactory.setScopDatabase(scop);
		
		String username = params.getUsername();
		jobs = new ArrayList<FarmJobRunnable>();
		for ( int i = 0 ; i < params.getThreads();i++){
			FarmJobRunnable.log("starting thread #" + (i+1));
			FarmJobRunnable runner = new FarmJobRunnable(params);	
			params.setUsername(username+"_thread_" + (i+1));
			jobs.add(runner);
			
			if ( progressListeners != null) {
				for (AlignmentProgressListener li : progressListeners){
					runner.addAlignmentProgressListener(li);
				}
{'fraction_non_alphanumeric': 0.11946641482705701, 'fraction_numerical': 0.003305394876637941, 'mean_word_length': 3.7595505617977527, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7153332', 'n_tokens_mistral': 2987, 'n_tokens_neox': 2533, 'n_words': 624}