<commit_before>/*
 * Copyright (c) 2015, SRI International
 * All rights reserved.
 * Licensed under the The BSD 3-Clause License;
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 * 
 * http://opensource.org/licenses/BSD-3-Clause
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 * 
 * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 * 
 * Neither the name of the aic-praise nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package com.sri.ai.praise.sgsolver.cli;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.StringJoiner;

import com.google.common.annotations.Beta;
import com.google.common.base.Charsets;
import com.sri.ai.praise.lang.ModelLanguage;
import com.sri.ai.praise.model.common.io.ModelPage;
import com.sri.ai.praise.model.common.io.PagedModelContainer;

import joptsimple.OptionException;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;

/**
 * Command line interface for running the SGSolver.
 * 
 * @author oreilly
 *
 */
@Beta
public class SGSolverCLI {
	public static final Charset FILE_CHARSET = Charsets.UTF_8;
	
	static class SGSolverArgs implements AutoCloseable {
		List<File>    inputFiles      = new ArrayList<>(); // --input    (required)
		ModelLanguage inputLanguage   = null;              // --language (optional)
		PrintStream   out             = System.out;        // --output   (optional)
		
		@Override
		public void close() throws IOException {
			out.flush();
			// Only close if not System.out
			if (out != System.out) {				
				out.close();
			}
		}
	}
	
	public static void main(String[] args) {
		try (SGSolverArgs solverArgs = getArgs(args)) {
// TODO
System.out.println("Input Model Language is "+solverArgs.inputLanguage);
		}
		catch (Exception ex) {
			System.err.println("Error calling SGSolver");
			ex.printStackTrace();
		}
	}
	
	//
	// PRIVATE
	//
	private static SGSolverArgs getArgs(String[] args) throws UnsupportedEncodingException, FileNotFoundException, IOException {
		SGSolverArgs result = new SGSolverArgs();
		
		OptionParser parser = new OptionParser();		
		// Required
		OptionSpec<File> input = parser.accepts("input", "input model file(s), use the '"+File.pathSeparator+"' character to separate files").withRequiredArg().required().ofType(File.class).withValuesSeparatedBy(File.pathSeparator);
		// Optional
		OptionSpec<String> language   = parser.accepts("language", "input model language (code), allowed values are "+getLegalModelLanguageCodesDescription()).withRequiredArg().ofType(String.class);
		OptionSpec<File>   outputFile = parser.accepts("output",   "output file name (defaults to stdout).").withRequiredArg().ofType(File.class);
		// Help
		OptionSpec<Void> help = parser.accepts("help", "command line options help").forHelp();
		
		List<String> errors   = new ArrayList<>();
		List<String> warnings = new ArrayList<>();
		try {
			OptionSet options = parser.parse(args);
			
			if (options.has(help)) {
				parser.printHelpOn(System.out);
				System.exit(0);
			}
			
			result.inputFiles.addAll(options.valuesOf(input));
			
			if (options.has(language)) {
				String languageCode = options.valueOf(language);
				result.inputLanguage = findLanguageModel(languageCode);
				if (result.inputLanguage == null) {
					errors.add("Input language code, "+languageCode+", is not legal. Legal values are "+getLegalModelLanguageCodesDescription()+".");
				}
			}
			else {
				// Guess the language from the input file names
				result.inputLanguage = guessLanguageModel(result.inputFiles);
				if (result.inputLanguage == null) {
					errors.add("Unable to guess the input language from the given input file name extensions. Legal input language codes are "+getLegalModelLanguageCodesDescription()+".");
				}
			}
			
			if (options.has(outputFile)) {
				File outFile = options.valueOf(outputFile);
				if (outFile.exists()) {
					if (outFile.isDirectory()) {
						errors.add("Output file specified ["+outFile.getAbsolutePath()+"] is a directory, must be a legal file name.");
					}
					else {
						warnings.add("Warning output file ["+outFile.getAbsolutePath()+"] already exists, will be overwritten.");
					}
				}
				// Only setup file output if there are no errors.
				if (errors.size() == 0) {
					result.out = new PrintStream(outFile, FILE_CHARSET.name());
				}
			}
		}
		catch (OptionException oe) {
			errors.add(oe.getMessage());
		}
		
		if (warnings.size() > 0) {
			warnings.forEach(warning -> System.err.println("WARNING: "+ warning));
		}
		
		if (errors.size() > 0) {
			errors.forEach(error -> System.err.println("ERROR: "+ error));
			parser.printHelpOn(System.err);
			System.exit(1);
		}
		
		return result;
	}
	
	private static String getLegalModelLanguageCodesDescription() {
		StringJoiner result = new StringJoiner(", ");
		Arrays.stream(ModelLanguage.values()).forEach(ml -> result.add(ml.getCode()));
		return result.toString();
	}
	
	private static ModelLanguage findLanguageModel(String languageCode) {
		ModelLanguage result = Arrays.stream(ModelLanguage.values()).filter(ml -> ml.getCode().toLowerCase().equals(languageCode.toLowerCase())).findFirst().orElse(null);
		
		return result;
	}
	
	private static ModelLanguage guessLanguageModel(List<File> inputFiles) {
		ModelLanguage result = Arrays.stream(ModelLanguage.values())
				.filter(ml -> inputFiles.stream().anyMatch(
						inputFile -> inputFile.getName().endsWith(ml.getDefaultFileExtension())))
				.findFirst().orElse(null);
		
		if (result == null) {
			// Check if the input is a container file and if so get the language from it
			result = inputFiles.stream()
						.filter(inputFile -> inputFile.getName().endsWith(PagedModelContainer.DEFAULT_CONTAINER_FILE_EXTENSION))
						.map(containerInputFile -> {
							ModelLanguage containedLanguage = null;
							try {
								List<ModelPage> models = PagedModelContainer.getModelPagesFromURI(containerInputFile.toURI());
								if (models.size() > 0) {
									containedLanguage = models.get(0).getLanguage();
								}
							}
							catch (IOException ioe) {
								System.err.println(ioe.getMessage());
								ioe.printStackTrace();
							}
							return containedLanguage;
						})
						.findFirst().orElse(null);
		}
		
		return result;
	}
}
<commit_msg>Update sg solver cli so that input model files are non-option arguments to the program.<commit_after>/*
 * Copyright (c) 2015, SRI International
 * All rights reserved.
 * Licensed under the The BSD 3-Clause License;
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 * 
 * http://opensource.org/licenses/BSD-3-Clause
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 * 
 * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 * 
 * Neither the name of the aic-praise nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package com.sri.ai.praise.sgsolver.cli;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.StringJoiner;

import com.google.common.annotations.Beta;
import com.google.common.base.Charsets;
import com.sri.ai.praise.lang.ModelLanguage;
import com.sri.ai.praise.model.common.io.ModelPage;
import com.sri.ai.praise.model.common.io.PagedModelContainer;

import joptsimple.OptionException;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;

/**
 * Command line interface for running the SGSolver.
 * 
 * @author oreilly
 *
 */
@Beta
public class SGSolverCLI {
	public static final Charset FILE_CHARSET = Charsets.UTF_8;
	
	static class SGSolverArgs implements AutoCloseable {
		List<File>    inputFiles      = new ArrayList<>(); // --input    (required)
		ModelLanguage inputLanguage   = null;              // --language (optional)
		PrintStream   out             = System.out;        // --output   (optional)
		
		@Override
		public void close() throws IOException {
			out.flush();
			// Only close if not System.out
			if (out != System.out) {				
				out.close();
			}
		}
	}
	
	public static void main(String[] args) {
		try (SGSolverArgs solverArgs = getArgs(args)) {
// TODO
System.out.println("Input Model Language is "+solverArgs.inputLanguage);
		}
		catch (Exception ex) {
			System.err.println("Error calling SGSolver");
			ex.printStackTrace();
		}
	}
	
	//
	// PRIVATE
	//
	private static SGSolverArgs getArgs(String[] args) throws UnsupportedEncodingException, FileNotFoundException, IOException {
		SGSolverArgs result = new SGSolverArgs();
		
		OptionParser parser = new OptionParser();		
		// Optional
		OptionSpec<String> language   = parser.accepts("language", "input model language (code), allowed values are "+getLegalModelLanguageCodesDescription()).withRequiredArg().ofType(String.class);
		OptionSpec<File>   outputFile = parser.accepts("output",   "output file name (defaults to stdout).").withRequiredArg().ofType(File.class);
		// Help
		OptionSpec<Void> help = parser.accepts("help", "command line options help").forHelp();
		//
		String commandLine = "java "+SGSolverCLI.class.getName() + " [--help] [--language language_code] [--output output_file_name] inputModelFile ...";
		
		List<String> errors   = new ArrayList<>();
		List<String> warnings = new ArrayList<>();
		try {
			OptionSet options = parser.parse(args);
			
			if (options.has(help)) {
				System.out.println(commandLine);
				parser.printHelpOn(System.out);
				System.exit(0);
			}
			for (Object inputFileName : options.nonOptionArguments()) {
				File inputFile = new File(inputFileName.toString());
				if (inputFile.exists()) {
					result.inputFiles.add(inputFile);
				}
				else {
					errors.add("Input file ["+inputFileName+"] is not a file or does not exist.");
				}
			}
			if (result.inputFiles.size() == 0) {
				errors.add("No input files specified");
			}
			
			if (options.has(language)) {
				String languageCode = options.valueOf(language);
				result.inputLanguage = findLanguageModel(languageCode);
				if (result.inputLanguage == null) {
					errors.add("Input language code, "+languageCode+", is not legal. Legal values are "+getLegalModelLanguageCodesDescription()+".");
				}
			}
			else {
				// Guess the language from the input file names
				result.inputLanguage = guessLanguageModel(result.inputFiles);
				if (result.inputLanguage == null) {
					errors.add("Unable to guess the input language from the given input file name extensions. Legal input language codes are "+getLegalModelLanguageCodesDescription()+".");
				}
			}
			
			if (options.has(outputFile)) {
				File outFile = options.valueOf(outputFile);
				if (outFile.exists()) {
					if (outFile.isDirectory()) {
						errors.add("Output file specified ["+outFile.getAbsolutePath()+"] is a directory, must be a legal file name.");
					}
					else {
						warnings.add("Warning output file ["+outFile.getAbsolutePath()+"] already exists, will be overwritten.");
					}
				}
				// Only setup file output if there are no errors.
				if (errors.size() == 0) {
					result.out = new PrintStream(outFile, FILE_CHARSET.name());
				}
			}
		}
		catch (OptionException oe) {
			errors.add(oe.getMessage());
		}
		
		if (warnings.size() > 0) {
			warnings.forEach(warning -> System.err.println("WARNING: "+ warning));
		}
		
		if (errors.size() > 0) {
			errors.forEach(error -> System.err.println("ERROR: "+ error));
			System.err.println(commandLine);
			parser.printHelpOn(System.err);
			System.exit(1);
		}
		
		return result;
	}
	
	private static String getLegalModelLanguageCodesDescription() {
		StringJoiner result = new StringJoiner(", ");
		Arrays.stream(ModelLanguage.values()).forEach(ml -> result.add(ml.getCode()));
		return result.toString();
	}
	
	private static ModelLanguage findLanguageModel(String languageCode) {
		ModelLanguage result = Arrays.stream(ModelLanguage.values()).filter(ml -> ml.getCode().toLowerCase().equals(languageCode.toLowerCase())).findFirst().orElse(null);
		
		return result;
	}
	
	private static ModelLanguage guessLanguageModel(List<File> inputFiles) {
		ModelLanguage result = Arrays.stream(ModelLanguage.values())
				.filter(ml -> inputFiles.stream().anyMatch(
						inputFile -> inputFile.getName().endsWith(ml.getDefaultFileExtension())))
				.findFirst().orElse(null);
		
		if (result == null) {
			// Check if the input is a container file and if so get the language from it
			result = inputFiles.stream()
						.filter(inputFile -> inputFile.getName().endsWith(PagedModelContainer.DEFAULT_CONTAINER_FILE_EXTENSION))
						.map(containerInputFile -> {
							ModelLanguage containedLanguage = null;
							try {
								List<ModelPage> models = PagedModelContainer.getModelPagesFromURI(containerInputFile.toURI());
								if (models.size() > 0) {
									containedLanguage = models.get(0).getLanguage();
								}
							}
							catch (IOException ioe) {
								System.err.println(ioe.getMessage());
								ioe.printStackTrace();
							}
							return containedLanguage;
						})
						.findFirst().orElse(null);
		}
		
		return result;
	}
}

{'fraction_non_alphanumeric': 0.11209091473638452, 'fraction_numerical': 0.0018009066633546543, 'mean_word_length': 3.7532467532467533, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4440111', 'n_tokens_mistral': 5310, 'n_tokens_neox': 4462, 'n_words': 1341}