<commit_before>package edu.ucdenver.ccp.util.file;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;

import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;

public class FileLoaderUtil {

	private static final String FILE_ENCODING_PROPERTY = "file.encoding";
	private static final String DEFAULT_ENCODING = System.getProperty(FILE_ENCODING_PROPERTY);

	private FileLoaderUtil() {
		// do not instantiate
	}

	/**
	 * Returns a List<String[]> containing the column values for the input file. One String[] per
	 * line of the file. If you want the entire line, set the delimiter to be null. If there is a
	 * delimiter specified, you must specify at least one column index to return.
	 * 
	 * @param inputFile
	 * @param delimiter
	 * @param commentIndicator
	 * @param columnIndexes
	 * @return
	 * @throws IOException
	 * @throws ArrayIndexOutOfBoundsException
	 * @throws IllegalArgumentException
	 */
	public static List<String[]> loadColumnsFromDelimitedFile(File inputFile, String delimiter,
			String commentIndicator, int... columnIndexes) throws IOException, ArrayIndexOutOfBoundsException,
			IllegalArgumentException {
		return loadColumnsFromDelimitedFile(inputFile, DEFAULT_ENCODING, delimiter, commentIndicator, columnIndexes);
	}

	/**
	 * Returns a List<String[]> containing the column values for the input file. One String[] per
	 * line of the file. If you want the entire line, set the delimiter to be null. If there is a
	 * delimiter specified, you must specify at least one column index to return.
	 * 
	 * @param inputFile
	 * @param encoding
	 * @param delimiter
	 * @param commentIndicator
	 * @param columnIndexes
	 * @return
	 * @throws IOException
	 * @throws ArrayIndexOutOfBoundsException
	 *             - if a requested column index does not exist
	 * @throws IllegalArgumentException
	 *             - if a delimiter is specified, but no column indexes are requested
	 */
	public static List<String[]> loadColumnsFromDelimitedFile(File inputFile, String encoding, String delimiter,
			String commentIndicator, int... columnIndexes) throws IOException, ArrayIndexOutOfBoundsException,
			IllegalArgumentException {
		if (delimiter != null && (columnIndexes == null || columnIndexes.length == 0)) {
			throw new IllegalArgumentException(String.format(
					"Cannot parse columns from line. A delimiter \"%s\" has been specified, "
							+ "however no columns have been requested. If you want the entire line, "
							+ "set the delimiter to be null.", delimiter));
		}
		List<String[]> outputColumns = new ArrayList<String[]>();
		for (Iterator<String> lineIter = getLineIterator(inputFile, encoding, commentIndicator); lineIter.hasNext();) {
			String line = lineIter.next();
			outputColumns.add(getColumnsFromLine(line, delimiter, columnIndexes));
		}
		return outputColumns;
	}

	/**
	 * Parses the input line and returns a String[] containing the requested columns of that line.
	 * If the delimiter is null or if there are no column indexes specified, the entire line is
	 * returned.
	 * 
	 * @param line
	 * @param delimiter
	 * @param columnIndexes
	 * @return
	 * @throws ArrayIndexOutOfBoundsException
	 */
	public static String[] getColumnsFromLine(String line, String delimiter, int... columnIndexes)
			throws ArrayIndexOutOfBoundsException {
		if (delimiter == null || columnIndexes == null || columnIndexes.length == 0) {
			return new String[] { line };
		} else {
			String[] outputColumns = new String[columnIndexes.length];
			String[] lineTokens = line.split(delimiter);
			int outputIndex = 0;
			for (int columnIndex : columnIndexes) {
				ensureColumnIndexIsValid(columnIndex, line, lineTokens);
				outputColumns[outputIndex++] = lineTokens[columnIndex];
			}
			return outputColumns;
		}
	}

	/**
	 * Returns a List<String> containing the contents of the requested column in the input file,
	 * assuming the file contains columns delimited by the delimiter String.
	 * 
	 * @param inputFile
	 * @param delimiter
	 * @param columnIndex
	 * @return
	 * @throws IOException
	 */
	public static List<String> loadColumnFromDelimitedFile(File inputFile, String encoding, String delimiter,
			String commentIndicator, int columnIndex) throws IOException {
		List<String[]> singleColumnAsArrayList = loadColumnsFromDelimitedFile(inputFile, encoding, delimiter,
				commentIndicator, columnIndex);
		List<String> outputList = new ArrayList<String>(singleColumnAsArrayList.size());
		while (!singleColumnAsArrayList.isEmpty()) {
			outputList.add(singleColumnAsArrayList.get(0)[0]);
			singleColumnAsArrayList.remove(0);
		}
		return outputList;
	}<commit_msg>Revised getColumnsFromLine() to work properly when no column indexes are provided, but a delimiter is.<commit_after>package edu.ucdenver.ccp.util.file;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;

import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;

import edu.ucdenver.ccp.util.collections.CollectionsUtil;

public class FileLoaderUtil {

	private static final String FILE_ENCODING_PROPERTY = "file.encoding";
	private static final String DEFAULT_ENCODING = System.getProperty(FILE_ENCODING_PROPERTY);

	private FileLoaderUtil() {
		// do not instantiate
	}

	/**
	 * Returns a List<String[]> containing the column values for the input file. One String[] per
	 * line of the file. If you want the entire line, set the delimiter to be null. If there is a
	 * delimiter specified, you must specify at least one column index to return.
	 * 
	 * @param inputFile
	 * @param delimiter
	 * @param commentIndicator
	 * @param columnIndexes
	 * @return
	 * @throws IOException
	 * @throws ArrayIndexOutOfBoundsException
	 * @throws IllegalArgumentException
	 */
	public static List<String[]> loadColumnsFromDelimitedFile(File inputFile, String delimiter,
			String commentIndicator, int... columnIndexes) throws IOException, ArrayIndexOutOfBoundsException,
			IllegalArgumentException {
		return loadColumnsFromDelimitedFile(inputFile, DEFAULT_ENCODING, delimiter, commentIndicator, columnIndexes);
	}

	/**
	 * Returns a List<String[]> containing the column values for the input file. One String[] per
	 * line of the file. If you want the entire line, set the delimiter to be null. If there is a
	 * delimiter specified, you must specify at least one column index to return. TODO: track the
	 * number of columns that are returned. If there is a line with a different number of columns
	 * than expected, throw an exception.
	 * 
	 * @param inputFile
	 * @param encoding
	 * @param delimiter
	 * @param commentIndicator
	 * @param columnIndexes
	 * @return
	 * @throws IOException
	 * @throws ArrayIndexOutOfBoundsException
	 *             - if a requested column index does not exist
	 * @throws IllegalArgumentException
	 *             - if a delimiter is specified, but no column indexes are requested
	 */
	public static List<String[]> loadColumnsFromDelimitedFile(File inputFile, String encoding, String delimiter,
			String commentIndicator, int... columnIndexes) throws IOException, ArrayIndexOutOfBoundsException,
			IllegalArgumentException {
		if (delimiter != null && (columnIndexes == null || columnIndexes.length == 0)) {
			throw new IllegalArgumentException(String.format(
					"Cannot parse columns from line. A delimiter \"%s\" has been specified, "
							+ "however no columns have been requested. If you want the entire line, "
							+ "set the delimiter to be null.", delimiter));
		}
		List<String[]> outputColumns = new ArrayList<String[]>();
		for (Iterator<String> lineIter = getLineIterator(inputFile, encoding, commentIndicator); lineIter.hasNext();) {
			String line = lineIter.next();
			outputColumns.add(getColumnsFromLine(line, delimiter, columnIndexes));
		}
		return outputColumns;
	}

	/**
	 * Parses the input line and returns a String[] containing the requested columns of that line.
	 * If the delimiter is null or if there are no column indexes specified, the entire line is
	 * returned.
	 * 
	 * @param line
	 * @param delimiter
	 * @param columnIndexes
	 * @return
	 * @throws ArrayIndexOutOfBoundsException
	 */
	public static String[] getColumnsFromLine(String line, String delimiter, int... columnIndexes)
			throws ArrayIndexOutOfBoundsException {
		if (delimiter == null) {
			return new String[] { line };
		} else {
			String[] lineTokens = line.split(delimiter);
			if (columnIndexes == null || columnIndexes.length == 0) {
				columnIndexes = CollectionsUtil.createZeroBasedSequence(lineTokens.length);
			}
			String[] outputColumns = new String[columnIndexes.length];
			int outputIndex = 0;
			for (int columnIndex : columnIndexes) {
				ensureColumnIndexIsValid(columnIndex, line, lineTokens);
				outputColumns[outputIndex++] = lineTokens[columnIndex];
			}
			return outputColumns;
		}
	}
	
	/**
	 * Returns a List<String> containing the contents of the requested column in the input file,
	 * assuming the file contains columns delimited by the delimiter String.
	 * 
	 * @param inputFile
	 * @param delimiter
	 * @param columnIndex
	 * @return
	 * @throws IOException
	 */
	public static List<String> loadColumnFromDelimitedFile(File inputFile, String encoding, String delimiter,
			String commentIndicator, int columnIndex) throws IOException {
		List<String[]> singleColumnAsArrayList = loadColumnsFromDelimitedFile(inputFile, encoding, delimiter,
				commentIndicator, columnIndex);
		List<String> outputList = new ArrayList<String>(singleColumnAsArrayList.size());
		while (!singleColumnAsArrayList.isEmpty()) {
			outputList.add(singleColumnAsArrayList.get(0)[0]);
			singleColumnAsArrayList.remove(0);
		}
		return outputList;
	}
{'fraction_non_alphanumeric': 0.08874610982833049, 'fraction_numerical': 0.0012046983234615, 'mean_word_length': 4.4288828337874655, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5825282', 'n_tokens_mistral': 2924, 'n_tokens_neox': 2638, 'n_words': 908}