<commit_before>package gov.va.med.srcalc.util.csv;

import gov.va.med.srcalc.util.ValidationCodes;

import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Optional;

/**
 * General-purpose class to parse a CSV file into an object per row.
 * @param <T> the type of object that represents each row
 */
public class CsvReader<T>
{
    private static final Logger fLogger = LoggerFactory.getLogger(CsvReader.class);
    
    private final RowTranslator<T> fRowTranslator;
    
    public CsvReader(final RowTranslator<T> rowTranslator)
    {
        fRowTranslator = rowTranslator;
    }

    /**
     * Parses the given CSVRecords from a CSV file into a List of objects.
     * @param records must be non-empty
     * @param errors for recording validation errors
     */
    private TabularParseResult<T> readProcedures(
            final List<CSVRecord> records, final Collection<TabularUploadError> errors)
    {
        int firstRow = 0;
        
        // Detect if the first row is a header and strip if so.
        if (fRowTranslator.isHeaderRow(records.get(0)))
        {
            fLogger.debug("Ignoring a header row in the CSV.");
            firstRow = 1;
        }

        // Parse each line and record errors.
        fLogger.trace(
                "About to translate {} records from CSV into Procedure objects.",
                records.size());
        final ArrayList<T> procedures = new ArrayList<>(records.size());
        for (int i = firstRow; i < records.size(); ++i)
        {
            final CSVRecord row = records.get(i);
            final Optional<T> rowObject = fRowTranslator.translateRow(row, errors, i + 1);
            // Per TabularParseResult, represent invalid objects as nulls.
            procedures.add(rowObject.orNull());
        }

        return new TabularParseResult<>(procedures, errors);
    }<commit_msg>ASRC-4: minor fixes to CsvReader<commit_after>package gov.va.med.srcalc.util.csv;

import gov.va.med.srcalc.util.ValidationCodes;

import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Optional;

/**
 * General-purpose class to parse a CSV file into an object per row.
 * @param <T> the type of object that represents each row
 */
public class CsvReader<T>
{
    private static final Logger fLogger = LoggerFactory.getLogger(CsvReader.class);
    
    private final RowTranslator<T> fRowTranslator;
    
    public CsvReader(final RowTranslator<T> rowTranslator)
    {
        fRowTranslator = rowTranslator;
    }

    /**
     * Parses the given CSVRecords from a CSV file into a List of objects.
     * @param records must be non-empty
     * @param errors for recording validation errors
     */
    private TabularParseResult<T> readProcedures(
            final List<CSVRecord> records, final Collection<TabularUploadError> errors)
    {
        int firstRow = 0;
        
        // Detect if the first row is a header and strip if so.
        if (fRowTranslator.isHeaderRow(records.get(0)))
        {
            fLogger.debug("Ignoring a header row in the CSV.");
            firstRow = 1;
        }

        fLogger.trace(
                "About to translate {} records from CSV into Procedure objects.",
                records.size() - firstRow);

        // Parse each line and record errors.
        final ArrayList<T> procedures = new ArrayList<>(records.size());
        for (int i = firstRow; i < records.size(); ++i)
        {
            final CSVRecord row = records.get(i);
            final Optional<T> rowObject = fRowTranslator.translateRow(row, errors, i + 1);
            // Per TabularParseResult, represent invalid objects as nulls.
            procedures.add(rowObject.orNull());
        }

        return new TabularParseResult<>(procedures, errors);
    }
{'fraction_non_alphanumeric': 0.09611356760530602, 'fraction_numerical': 0.003025366534791715, 'mean_word_length': 2.642372881355932, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3226767', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1239, 'n_words': 361}