<commit_before>package com.nhl.link.move.runtime.csv;

import com.nhl.link.move.LmRuntimeException;
import com.nhl.link.move.Row;
import com.nhl.link.move.RowAttribute;
import com.nhl.link.move.RowReader;

import java.util.Iterator;
import java.util.List;

/**
 * @since 1.4
 */
public class CsvRowReader implements RowReader {

    public static final char DEFAULT_DELIMITER = ',';

    private RowAttribute[] attributes;
    private List<String> data;
    private char delimiter;
    private Integer readFrom;

    public CsvRowReader(RowAttribute[] attributes, List<String> data) {
        this.attributes = attributes;
        this.data = data;
        this.delimiter = DEFAULT_DELIMITER;
        this.readFrom = 1;
    }

    public void setDelimiter(String delimiter) {
        if (delimiter.length() != 1) {
            throw new LmRuntimeException("Invalid delimiter (should be exactly one character): " + delimiter);
        }
        this.delimiter = delimiter.charAt(0);
    }

    public void setReadFrom(Integer readFrom) {
        if (readFrom <= 0) {
            throw new LmRuntimeException("Invalid line number: " + readFrom);
        }
        this.readFrom = readFrom;
    }

    @Override
    public Iterator<Row> iterator() {
        final Iterator<String> drIt = data.iterator();

        // wind forward if needed
        int i = 0;
        while (++i < readFrom) {
            drIt.next();
        }

        return new Iterator<Row>() {
            @Override
            public boolean hasNext() {
                return drIt.hasNext();
            }

            @Override
            public Row next() {
                return new CsvDataRow(attributes, drIt.next(), delimiter);
            }

            @Override
            public void remove() {
                throw new UnsupportedOperationException();
            }
        };
    }

    @Override
    public void close() {
        // do nothing
    }
}
<commit_msg>#56 Support for non printable delimiter characters in CSV extractor<commit_after>package com.nhl.link.move.runtime.csv;

import com.nhl.link.move.LmRuntimeException;
import com.nhl.link.move.Row;
import com.nhl.link.move.RowAttribute;
import com.nhl.link.move.RowReader;

import java.util.Iterator;
import java.util.List;

/**
 * @since 1.4
 */
public class CsvRowReader implements RowReader {

    public static final char DEFAULT_DELIMITER = ',';

    private RowAttribute[] attributes;
    private List<String> data;
    private char delimiter;
    private Integer readFrom;

    public CsvRowReader(RowAttribute[] attributes, List<String> data) {
        this.attributes = attributes;
        this.data = data;
        this.delimiter = DEFAULT_DELIMITER;
        this.readFrom = 1;
    }

    public void setDelimiter(String delimiter) {
        if (delimiter.startsWith("0x")) {
            // we have a character code here
            int codePoint = Integer.valueOf(delimiter.substring(2, delimiter.length()), 16);
            if (Character.isValidCodePoint(codePoint) && Character.charCount(codePoint) == 1) {
                this.delimiter = (char) codePoint;
            } else {
                throw new LmRuntimeException("Invalid delimiter (not a valid Unicode code point): "
                        + delimiter);
            }
        } else if (delimiter.length() != 1) {
            throw new LmRuntimeException("Invalid delimiter (should be exactly one character): " + delimiter);
        }
        this.delimiter = delimiter.charAt(0);
    }

    public void setReadFrom(Integer readFrom) {
        if (readFrom <= 0) {
            throw new LmRuntimeException("Invalid line number: " + readFrom);
        }
        this.readFrom = readFrom;
    }

    @Override
    public Iterator<Row> iterator() {
        final Iterator<String> drIt = data.iterator();

        // wind forward if needed
        int i = 0;
        while (++i < readFrom) {
            drIt.next();
        }

        return new Iterator<Row>() {
            @Override
            public boolean hasNext() {
                return drIt.hasNext();
            }

            @Override
            public Row next() {
                return new CsvDataRow(attributes, drIt.next(), delimiter);
            }

            @Override
            public void remove() {
                throw new UnsupportedOperationException();
            }
        };
    }

    @Override
    public void close() {
        // do nothing
    }
}

{'fraction_non_alphanumeric': 0.10194936141608783, 'fraction_numerical': 0.0047053551422809765, 'mean_word_length': 2.010114632501686, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6201160', 'n_tokens_mistral': 1258, 'n_tokens_neox': 1216, 'n_words': 312}