<commit_before>/*******************************************************************************
 * Copyright (c) 2013, 2014- UT-Battelle, LLC.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *   Initial API and implementation and/or initial documentation - <NAME>,
 *   <NAME>, <NAME>, <NAME>, <NAME>,
 *   <NAME>, <NAME>, <NAME>
 *******************************************************************************/
package org.eclipse.ice.item.utilities.moose;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.URI;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Stack;

import org.eclipse.core.resources.IFile;
import org.eclipse.ice.datastructures.form.AdaptiveTreeComposite;
import org.eclipse.ice.datastructures.form.DataComponent;
import org.eclipse.ice.datastructures.form.Entry;
import org.eclipse.ice.datastructures.form.Form;
import org.eclipse.ice.datastructures.form.TreeComposite;
import org.eclipse.ice.datastructures.form.iterator.BreadthFirstTreeCompositeIterator;
import org.eclipse.ice.datastructures.ICEObject.Component;
import org.eclipse.ice.io.serializable.IReader;
import org.eclipse.ice.io.serializable.IWriter;
import org.eclipse.ice.item.nuclear.MOOSEModel;

import java.util.Map;

import javax.naming.OperationNotSupportedException;

import org.yaml.snakeyaml.Yaml;

/**
 * <p>
 * This class reads and writes MOOSE Blocks and Parameters to and from the
 * different MOOSE file types, including parsing from YAML and writing to
 * GetPot.
 * 
 * There are two primary types of files associated with MOOSE: the YAML file
 * used to specify the possible configuration of an input file and the input
 * file itself, which is a GetPot/Perl configuration file.
 * 
 * This class realizes the IComponentVisitor interface to find DataComponents in
 * the TreeComposites that can be converted into a parameter set for a MOOSE
 * input block. If there are other Components in a TreeComposite. They are
 * completely ignored.
 * 
 * Blocks and YAMLBlocks are used because each block needs to be converted to or
 * from a TreeComposite. This is complicated in the case of loading the YAML
 * input specification because it is a *specification* (or schema) and not the
 * input itself. The nodes of this tree are what could be configured, not what
 * is, so they must be setup as child exemplars on a TreeComposite.
 * </p>
 * 
 * @author <NAME>, <NAME>, <NAME>
 */
public class MOOSEFileHandler implements IReader, IWriter {

	/**
	 * A flag to denote whether or not debugging is enabled
	 */
	private static boolean debugFlag = false;

	/**
	 * Set the debug flag
	 */
	static {
		if (System.getProperty("DebugICE") != null) {
			debugFlag = true;
		}
	}

	/**
	 * This operation writes a set of MOOSE blocks to the specified file path.
	 * 
	 * @param filePath
	 *            The file path to which the MOOSE blocks should be dumped. If
	 *            the path is null or empty, the operation returns without doing
	 *            any work.
	 * @param blockSet
	 *            The collection of TreeComposites that represent MOOSE blocks
	 *            to be dumped to the file. The TreeComposites should only
	 *            contain a single DataComponent, id = 1, and other
	 *            TreeComposites. Any other components in the TreeComposite will
	 *            be ignored.        
	 */
	public void dumpInputFile(String filePath, ArrayList<TreeComposite> blockSet) {

		// Local Declarations
		File inputFile = null;
		String outputString = "";
		FileWriter fileWriter = null;
		BufferedWriter fileOutputWriter = null;
		ArrayList<Block> blocks = null;

		// Only do this if the file path and data are valid
		if (filePath != null && blockSet != null) {
			// Create the blocks from the incoming tree composites
			blocks = new ArrayList<Block>();
			for (TreeComposite blockTree : blockSet) {
				Block block = new Block();
				block.fromTreeComposite(blockTree);
				blocks.add(block);
			}
			// Reorganize the blocks so that blocks that must come first, such
			// as "Functions" and "Variables" do.
			for (int i = 0; i < blocks.size(); i++) {
				// Grab the block and its name
				Block block = blocks.get(i);
				String name = block.getName();
				// Check for the names of the blocks that have to be written
				// first
				if ("Functions".equals(name) || "Variables".equals(name)) {
					// Remove the block from the set
					blocks.remove(i);
					// Write the block to the output string
					outputString += block.toGetPot(null);
					// Decrement the counter so we go back and get any remaining
					// blocks.
					--i;
				}
			}
			// Dump the blocks to the output String
			for (int i = 0; i < blocks.size(); i++) {
				outputString += blocks.get(i).toGetPot(null);
			}
			// Try to write the file
			try {
				// Open the file and writers
				inputFile = new File(filePath);
				fileWriter = new FileWriter(inputFile);
				fileOutputWriter = new BufferedWriter(fileWriter);
				// Dump the string to the file
				fileOutputWriter.write(outputString);
				// Flush everything to the file and close it
				fileOutputWriter.flush();
				fileOutputWriter.close();
				fileWriter.close();
			} catch (IOException e) {
				System.out.println("MOOSEFileHandler Exception: "
						+ "Unable to write output file.");
				e.printStackTrace();
			}
		}

		return;
	}

	/**
	 * This operations loads a MOOSE GetPot file at the specified path and
	 * returns a fully-configured set of ICE TreeComposites.
	 * 
	 * @param filePath
	 *            The file path from which the MOOSE blocks written in GetPot
	 *            should be read. If the path is null or empty, the operation
	 *            returns without doing any work.
	 * @return The MOOSE input file specification as read from the GetPot input
	 *         and stored in TreeComposites. Each TreeComposite contains both
	 *         parameters and exemplar children. Any parameters in a
	 *         TreeComposite are contained in a DataComponent. The id of the
	 *         data component is 1.
	 */
	public ArrayList<TreeComposite> loadFromGetPot(String filePath) {

		// Local Declarations
		ArrayList<TreeComposite> trees = new ArrayList<TreeComposite>();
		byte[] fileByteArray = null;
		String mooseFileString = null, potLine = null;

		// Quit if the path is boned
		if (filePath == null || filePath.isEmpty()) {
			return null;
		}

		// Post some debug info
		if (debugFlag) {
			System.out.println("MOOSEFileHandler Message: "
					+ "Attempting to loading GetPot file " + filePath);
		}

		// Load the GetPot file
		try {
			RandomAccessFile mooseFile = new RandomAccessFile(filePath, "r");
			// Put it into a byte array
			fileByteArray = new byte[(int) mooseFile.length()];
			mooseFile.read(fileByteArray);
			// And then a string
			mooseFileString = new String(fileByteArray);
			// Close the file
			mooseFile.close();
			// Post some more debug info
			if (debugFlag) {
				System.out.println("MOOSEFileHandler Message: File loaded.");
			}
		} catch (IOException e) {
			// Complain if the file is not found
			System.err.println("MOOSEFileHandler Message: "
					+ "Unable to load GetPot file!");
			e.printStackTrace();
		}

		// Check the string before proceeding
		if (mooseFileString != null && !mooseFileString.isEmpty()) {
			// Create an array list from the string
			ArrayList<String> potLines = new ArrayList<String>(
					Arrays.asList(mooseFileString.split("\n")));

			// Remove (non-parameter) commented lines and white space
			String trimmedPotLine = "";
			for (int i = 0; i < potLines.size(); i++) {

				trimmedPotLine = potLines.get(i).trim();

				if (trimmedPotLine.startsWith("#")
						&& !trimmedPotLine.contains("=")
						&& !trimmedPotLine.contains("[")
						&& !trimmedPotLine.contains("]")) {
					// Lines that start with "#" but have no "=" are comments
					// that aren't parameters and should be removed
					potLines.remove(i);
					// Update "i" so that we read correctly
					--i;
				} else if (potLines.get(i).isEmpty()) {
					// Remove empty lines
					potLines.remove(i);
					// Update "i" so that we read correctly
					--i;
				} else {
					// All other lines should be trimmed
					potLines.set(i, potLines.get(i).trim());
				}
			}

			// Read all of the lines again, create blocks and load them.
			int counter = 0, endCounter = 1;
			while (counter < potLines.size()) {
				// Get the line and shift the counters
				potLine = potLines.get(counter);
				++counter;

				// The start of a full block is a line with the name in brackets
				// and without the "./" sequence.
				if (potLine.contains("[") && potLine.contains("]")) {
					// Go to the next line
					potLine = potLines.get(endCounter);

					// Loop over the block and find the end
					while (!potLine.contains("[]")) {
						// Update the line and the counter
						potLine = potLines.get(endCounter);
						++endCounter;
					}
					// Create a new block
					Block block = new Block();
					ArrayList<String> blockLines = new ArrayList<String>(
							potLines.subList(counter - 1, endCounter));
					StringBuilder stringBuilder = new StringBuilder(
							blockLines.get(0));
					blockLines.set(0, stringBuilder.toString());
					block.fromGetPot(blockLines);
					// Add the block to the list
					trees.add(block.toTreeComposite());
					// Update the counter to point to the last read line
					counter = endCounter;
					// Print some debug information
					if (debugFlag) {
						System.out.println("\nMOOSEFileHandler Message: "
								+ "Block output read from GetPot file "
								+ filePath + " follows.");
						// Dump each line of the newly created block
						for (String line : blockLines) {
							System.out.println(line);
						}
					}
				}
			}
		} else if (debugFlag) {
			System.err.println("MOOSEFileHandler Message: "
					+ "String loaded from " + filePath + " is null or empty.");
		}

		return trees;
	}

	/**
	 * This operations loads a MOOSE YAML file at the specified path and returns
	 * a fully-configured set of ICE TreeComposites.
	 * 
	 * @param filePath
	 *            The file path from which the MOOSE blocks written in YAML
	 *            should be read. If the path is null or empty, the operation
	 *            returns without doing any work.
	 * @return The MOOSE input file specification as read from the YAML input
	 *         and stored in TreeComposites. Each TreeComposite contains both
	 *         parameters and exemplar children. Any parameters in a
	 *         TreeComposite are contained in a DataComponent. The id of the
	 *         data component is 1.
	 * @throws IOException
	 */
	public ArrayList<TreeComposite> loadYAML(String filePath)
			throws IOException {<commit_msg>Resolved a conflicting file in merge.<commit_after>/*******************************************************************************
 * Copyright (c) 2013, 2014- UT-Battelle, LLC.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *   Initial API and implementation and/or initial documentation - <NAME>,
 *   <NAME>, <NAME>, <NAME>, <NAME>,
 *   <NAME>, <NAME>, <NAME>
 *******************************************************************************/
package org.eclipse.ice.item.utilities.moose;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.URI;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Stack;

import org.eclipse.core.resources.IFile;
import org.eclipse.ice.datastructures.form.AdaptiveTreeComposite;
import org.eclipse.ice.datastructures.form.DataComponent;
import org.eclipse.ice.datastructures.form.Entry;
import org.eclipse.ice.datastructures.form.Form;
import org.eclipse.ice.datastructures.form.TreeComposite;
import org.eclipse.ice.datastructures.form.iterator.BreadthFirstTreeCompositeIterator;
import org.eclipse.ice.datastructures.ICEObject.Component;
import org.eclipse.ice.io.serializable.IReader;
import org.eclipse.ice.io.serializable.IWriter;
import org.eclipse.ice.item.nuclear.MOOSEModel;

import java.util.Map;

import javax.naming.OperationNotSupportedException;

import org.yaml.snakeyaml.Yaml;

/**
 * <p>
 * This class reads and writes MOOSE Blocks and Parameters to and from the
 * different MOOSE file types, including parsing from YAML and writing to
 * GetPot.
 * </p>
 * <p>
 * There are two primary types of files associated with MOOSE: the YAML file
 * used to specify the possible configuration of an input file and the input
 * file itself, which is a GetPot/Perl configuration file.
 * </p>
 * <p>
 * This class realizes the IComponentVisitor interface to find DataComponents in
 * the TreeComposites that can be converted into a parameter set for a MOOSE
 * input block. If there are other Components in a TreeComposite. They are
 * completely ignored.
 * </p>
 * <p>
 * Blocks and YAMLBlocks are used because each block needs to be converted to or
 * from a TreeComposite. This is complicated in the case of loading the YAML
 * input specification because it is a *specification* (or schema) and not the
 * input itself. The nodes of this tree are what could be configured, not what
 * is, so they must be setup as child exemplars on a TreeComposite.
 * </p>
 * 
 * @author <NAME>, <NAME>, <NAME>
 */
public class MOOSEFileHandler implements IReader, IWriter {

	/**
	 * A flag to denote whether or not debugging is enabled
	 */
	private static boolean debugFlag = false;

	/**
	 * Set the debug flag
	 */
	static {
		if (System.getProperty("DebugICE") != null) {
			debugFlag = true;
		}
	}

	/**
	 * This operation writes a set of MOOSE blocks to the specified file path.
	 * 
	 * @param filePath
	 *            The file path to which the MOOSE blocks should be dumped. If
	 *            the path is null or empty, the operation returns without doing
	 *            any work.
	 * @param blockSet
	 *            The collection of TreeComposites that represent MOOSE blocks
	 *            to be dumped to the file. The TreeComposites should only
	 *            contain a single DataComponent, id = 1, and other
	 *            TreeComposites. Any other components in the TreeComposite will
	 *            be ignored.        
	 */
	public void dumpInputFile(String filePath, ArrayList<TreeComposite> blockSet) {

		// Local Declarations
		File inputFile = null;
		String outputString = "";
		FileWriter fileWriter = null;
		BufferedWriter fileOutputWriter = null;
		ArrayList<Block> blocks = null;

		// Only do this if the file path and data are valid
		if (filePath != null && blockSet != null) {
			// Create the blocks from the incoming tree composites
			blocks = new ArrayList<Block>();
			for (TreeComposite blockTree : blockSet) {
				Block block = new Block();
				block.fromTreeComposite(blockTree);
				blocks.add(block);
			}
			// Reorganize the blocks so that blocks that must come first, such
			// as "Functions" and "Variables" do.
			for (int i = 0; i < blocks.size(); i++) {
				// Grab the block and its name
				Block block = blocks.get(i);
				String name = block.getName();
				// Check for the names of the blocks that have to be written
				// first
				if ("Functions".equals(name) || "Variables".equals(name)) {
					// Remove the block from the set
					blocks.remove(i);
					// Write the block to the output string
					outputString += block.toGetPot(null);
					// Decrement the counter so we go back and get any remaining
					// blocks.
					--i;
				}
			}
			// Dump the blocks to the output String
			for (int i = 0; i < blocks.size(); i++) {
				outputString += blocks.get(i).toGetPot(null);
			}
			// Try to write the file
			try {
				// Open the file and writers
				inputFile = new File(filePath);
				fileWriter = new FileWriter(inputFile);
				fileOutputWriter = new BufferedWriter(fileWriter);
				// Dump the string to the file
				fileOutputWriter.write(outputString);
				// Flush everything to the file and close it
				fileOutputWriter.flush();
				fileOutputWriter.close();
				fileWriter.close();
			} catch (IOException e) {
				System.out.println("MOOSEFileHandler Exception: "
						+ "Unable to write output file.");
				e.printStackTrace();
			}
		}

		return;
	}

	/**
	 * This operations loads a MOOSE GetPot file at the specified path and
	 * returns a fully-configured set of ICE TreeComposites.
	 * 
	 * @param filePath
	 *            The file path from which the MOOSE blocks written in GetPot
	 *            should be read. If the path is null or empty, the operation
	 *            returns without doing any work.
	 * @return The MOOSE input file specification as read from the GetPot input
	 *         and stored in TreeComposites. Each TreeComposite contains both
	 *         parameters and exemplar children. Any parameters in a
	 *         TreeComposite are contained in a DataComponent. The id of the
	 *         data component is 1.
	 */
	public ArrayList<TreeComposite> loadFromGetPot(String filePath) {

		// Local Declarations
		ArrayList<TreeComposite> trees = new ArrayList<TreeComposite>();
		byte[] fileByteArray = null;
		String mooseFileString = null, potLine = null;

		// Quit if the path is boned
		if (filePath == null || filePath.isEmpty()) {
			return null;
		}

		// Post some debug info
		if (debugFlag) {
			System.out.println("MOOSEFileHandler Message: "
					+ "Attempting to loading GetPot file " + filePath);
		}

		// Load the GetPot file
		try {
			RandomAccessFile mooseFile = new RandomAccessFile(filePath, "r");
			// Put it into a byte array
			fileByteArray = new byte[(int) mooseFile.length()];
			mooseFile.read(fileByteArray);
			// And then a string
			mooseFileString = new String(fileByteArray);
			// Close the file
			mooseFile.close();
			// Post some more debug info
			if (debugFlag) {
				System.out.println("MOOSEFileHandler Message: File loaded.");
			}
		} catch (IOException e) {
			// Complain if the file is not found
			System.err.println("MOOSEFileHandler Message: "
					+ "Unable to load GetPot file!");
			e.printStackTrace();
		}

		// Check the string before proceeding
		if (mooseFileString != null && !mooseFileString.isEmpty()) {
			// Create an array list from the string
			ArrayList<String> potLines = new ArrayList<String>(
					Arrays.asList(mooseFileString.split("\n")));

			// Remove (non-parameter) commented lines and white space
			String trimmedPotLine = "";
			for (int i = 0; i < potLines.size(); i++) {

				trimmedPotLine = potLines.get(i).trim();

				if (trimmedPotLine.startsWith("#")
						&& !trimmedPotLine.contains("=")
						&& !trimmedPotLine.contains("[")
						&& !trimmedPotLine.contains("]")) {
					// Lines that start with "#" but have no "=" are comments
					// that aren't parameters and should be removed
					potLines.remove(i);
					// Update "i" so that we read correctly
					--i;
				} else if (potLines.get(i).isEmpty()) {
					// Remove empty lines
					potLines.remove(i);
					// Update "i" so that we read correctly
					--i;
				} else {
					// All other lines should be trimmed
					potLines.set(i, potLines.get(i).trim());
				}
			}

			// Read all of the lines again, create blocks and load them.
			int counter = 0, endCounter = 1;
			while (counter < potLines.size()) {
				// Get the line and shift the counters
				potLine = potLines.get(counter);
				++counter;

				// The start of a full block is a line with the name in brackets
				// and without the "./" sequence.
				if (potLine.contains("[") && potLine.contains("]")) {
					// Go to the next line
					potLine = potLines.get(endCounter);

					// Loop over the block and find the end
					while (!potLine.contains("[]")) {
						// Update the line and the counter
						potLine = potLines.get(endCounter);
						++endCounter;
					}
					// Create a new block
					Block block = new Block();
					ArrayList<String> blockLines = new ArrayList<String>(
							potLines.subList(counter - 1, endCounter));
					StringBuilder stringBuilder = new StringBuilder(
							blockLines.get(0));
					blockLines.set(0, stringBuilder.toString());
					block.fromGetPot(blockLines);
					// Add the block to the list
					trees.add(block.toTreeComposite());
					// Update the counter to point to the last read line
					counter = endCounter;
					// Print some debug information
					if (debugFlag) {
						System.out.println("\nMOOSEFileHandler Message: "
								+ "Block output read from GetPot file "
								+ filePath + " follows.");
						// Dump each line of the newly created block
						for (String line : blockLines) {
							System.out.println(line);
						}
					}
				}
			}
		} else if (debugFlag) {
			System.err.println("MOOSEFileHandler Message: "
					+ "String loaded from " + filePath + " is null or empty.");
		}

		return trees;
	}

	/**
	 * This operations loads a MOOSE YAML file at the specified path and returns
	 * a fully-configured set of ICE TreeComposites.
	 * 
	 * @param filePath
	 *            The file path from which the MOOSE blocks written in YAML
	 *            should be read. If the path is null or empty, the operation
	 *            returns without doing any work.
	 * @return
	 *         The MOOSE input file specification as read from the YAML input
	 *         and stored in TreeComposites. Each TreeComposite contains both
	 *         parameters and exemplar children. Any parameters in a
	 *         TreeComposite are contained in a DataComponent. The id of the
	 *         data component is 1.
	 * @throws IOException
	 */
	public ArrayList<TreeComposite> loadYAML(String filePath)
			throws IOException {
{'fraction_non_alphanumeric': 0.11073960144442958, 'fraction_numerical': 0.002050733360082029, 'mean_word_length': 2.985785358919687, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '834623', 'n_tokens_mistral': 7483, 'n_tokens_neox': 6233, 'n_words': 2299}