<commit_before>package uk.ac.ebi.embl.template;

import org.apache.commons.lang.StringUtils;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

public class CSVReader {
    private String currentLine;
    private final BufferedReader lineReader;
    private List<String> headerKeys;
    private int lineNumber = 0;
    private int entryNumber = 0;
    private Set<Integer> entryNumbers = new HashSet<>();

    public CSVReader(final InputStream inputReader,final List<TemplateTokenInfo> allTokens, final int expectedMatchNumber) throws Exception {
        lineReader = new BufferedReader(new InputStreamReader(inputReader));
        readHeader(expectedMatchNumber, allTokens);
    }

    public CSVLine readTemplateSpreadsheetLine() throws Exception {
        skipEmptyLinesAndComments();
        if (currentLine != null) {
            return processTemplateSpreadsheetLine();
        }
        return null;
    }

    public void skipEmptyLinesAndComments() throws Exception {
        // Skip empty lines and comment lines.
        while(currentLine != null && (
              currentLine.isEmpty() ||
              currentLine.startsWith(FastaSpreadsheetConverter.COMMENT_TOKEN))) {
            readLine();
        }
    }

    public CSVLine processTemplateSpreadsheetLine() throws Exception {
        final TemplateVariables entryTokensMap = new TemplateVariables();
        final String[] currentTokenLine = StringUtils.splitPreserveAllTokens(currentLine, CSVWriter.UPLOAD_DELIMITER);
        if ((currentTokenLine.length) != headerKeys.size()) {
            String lineSummary = currentLine;
            if (currentLine.length() > 10)
                lineSummary = currentLine.substring(0, 10);
            throw new TemplateUserError("There are " + headerKeys.size() + " tokens specified in the header but " + currentTokenLine.length + " values for entry on line " + lineSummary + "..., please check your import file data is properly delimited with a 'tab'.");
        }

        // The entry number will be used as the sequence name and must be unique
        // for a submission.
        entryNumber++;
        if (entryNumbers.contains(entryNumber)) {
            throw new TemplateUserError("Internal error: non-unique sequence number error" );
        }
        entryNumbers.add(entryNumber);

        entryTokensMap.setSequenceName( String.valueOf(entryNumber));

        for (int i = 1; i < currentTokenLine.length; i++) {
            String tokenValue = currentTokenLine[i];
            checkTokenForBannedCharacters(tokenValue);
            tokenValue = tokenValue.replaceAll("<br>", "\n");
            tokenValue = tokenValue.replaceAll(";", ",");
            if (tokenValue.startsWith("\"") && tokenValue.endsWith("\"")) {
                tokenValue = StringUtils.stripStart(tokenValue, "\"");
                tokenValue = StringUtils.stripEnd(tokenValue, "\"");
            }
            entryTokensMap.addToken(headerKeys.get(i), tokenValue);
        }
        readLine();
        return new CSVLine(++lineNumber, entryTokensMap);
    }

    private void checkTokenForBannedCharacters(final String tokenValue) throws Exception {
        if (!StringUtils.isBlank(tokenValue)) {
            if (tokenValue.contains(TemplateProcessorConstants.DELIMITER1) ||
                    tokenValue.contains(TemplateProcessorConstants.DELIMITER2) ||
                    tokenValue.contains(TemplateProcessorConstants.DELIMITER3) ||
                    tokenValue.contains(TemplateProcessorConstants.DELIMITER4)) {
                final String message = "Contains illegal characters <d1>, <d2>, <d3> or <d4>";
                throw new TemplateUserError(message);
            }
        }
    }

    private void prepareLineForParsing() {
        if (currentLine == null) {
            return;
        }
        currentLine = currentLine.trim();
        if (currentLine.startsWith("\""))
            currentLine = currentLine.replaceFirst("\"", "");//get rid of starting " if present - open office puts these in for strings and they need to be removed
        if(currentLine.endsWith(";"))
            currentLine = StringUtils.removeEnd(currentLine, ";");
    }

    private void readHeader(final int expectedMatchNumber, final List<TemplateTokenInfo> allTokens) throws Exception {
        readLine();
        if (currentLine == null) {
            throw new TemplateException("Template file is empty");
        }
        String header = null;
        skipEmptyLinesAndComments();
        if (currentLine != null) {
            header = currentLine;
            readLine();
        }
        if (header == null) {
            throw new TemplateUserError("Template file has no data");
        }
        header = header.replaceAll("\"", "");//remove all speech marks - open office puts these in
        final String[] headerTokens = header.split(CSVWriter.UPLOAD_DELIMITER);
        final List<String> recognizedKeys = new ArrayList<>();
        headerKeys = new ArrayList<>();<commit_msg>Added check based on organism name fields that header line exists.<commit_after>package uk.ac.ebi.embl.template;

import org.apache.commons.lang.StringUtils;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import static uk.ac.ebi.embl.template.TemplateProcessorConstants.ORGANISM_NAME_TOKEN;
import static uk.ac.ebi.embl.template.TemplateProcessorConstants.ORGANISM_TOKEN;

public class CSVReader {
    private String currentLine;
    private final BufferedReader lineReader;
    private List<String> headerKeys;
    private int lineNumber = 0;
    private int entryNumber = 0;
    private Set<Integer> entryNumbers = new HashSet<>();

    public CSVReader(final InputStream inputReader,final List<TemplateTokenInfo> allTokens, final int expectedMatchNumber) throws Exception {
        lineReader = new BufferedReader(new InputStreamReader(inputReader));
        readHeader(expectedMatchNumber, allTokens);
    }

    public CSVLine readTemplateSpreadsheetLine() throws Exception {
        skipEmptyLinesAndComments();
        if (currentLine != null) {
            return processTemplateSpreadsheetLine();
        }
        return null;
    }

    public void skipEmptyLinesAndComments() throws Exception {
        // Skip empty lines and comment lines.
        while(currentLine != null && (
              currentLine.isEmpty() ||
              currentLine.startsWith(FastaSpreadsheetConverter.COMMENT_TOKEN))) {
            readLine();
        }
    }

    public CSVLine processTemplateSpreadsheetLine() throws Exception {
        final TemplateVariables entryTokensMap = new TemplateVariables();
        final String[] currentTokenLine = StringUtils.splitPreserveAllTokens(currentLine, CSVWriter.UPLOAD_DELIMITER);
        if ((currentTokenLine.length) != headerKeys.size()) {
            String lineSummary = currentLine;
            if (currentLine.length() > 10)
                lineSummary = currentLine.substring(0, 10);
            throw new TemplateUserError("There are " + headerKeys.size() + " tokens specified in the header but " + currentTokenLine.length + " values for entry on line " + lineSummary + "..., please check your import file data is properly delimited with a 'tab'.");
        }

        // The entry number will be used as the sequence name and must be unique
        // for a submission.
        entryNumber++;
        if (entryNumbers.contains(entryNumber)) {
            throw new TemplateUserError("Internal error: non-unique sequence number error" );
        }
        entryNumbers.add(entryNumber);

        entryTokensMap.setSequenceName( String.valueOf(entryNumber));

        for (int i = 1; i < currentTokenLine.length; i++) {
            String tokenValue = currentTokenLine[i];
            checkTokenForBannedCharacters(tokenValue);
            tokenValue = tokenValue.replaceAll("<br>", "\n");
            tokenValue = tokenValue.replaceAll(";", ",");
            if (tokenValue.startsWith("\"") && tokenValue.endsWith("\"")) {
                tokenValue = StringUtils.stripStart(tokenValue, "\"");
                tokenValue = StringUtils.stripEnd(tokenValue, "\"");
            }
            entryTokensMap.addToken(headerKeys.get(i), tokenValue);
        }
        readLine();
        return new CSVLine(++lineNumber, entryTokensMap);
    }

    private void checkTokenForBannedCharacters(final String tokenValue) throws Exception {
        if (!StringUtils.isBlank(tokenValue)) {
            if (tokenValue.contains(TemplateProcessorConstants.DELIMITER1) ||
                    tokenValue.contains(TemplateProcessorConstants.DELIMITER2) ||
                    tokenValue.contains(TemplateProcessorConstants.DELIMITER3) ||
                    tokenValue.contains(TemplateProcessorConstants.DELIMITER4)) {
                final String message = "Contains illegal characters <d1>, <d2>, <d3> or <d4>";
                throw new TemplateUserError(message);
            }
        }
    }

    private void prepareLineForParsing() {
        if (currentLine == null) {
            return;
        }
        currentLine = currentLine.trim();
        if (currentLine.startsWith("\""))
            currentLine = currentLine.replaceFirst("\"", "");//get rid of starting " if present - open office puts these in for strings and they need to be removed
        if(currentLine.endsWith(";"))
            currentLine = StringUtils.removeEnd(currentLine, ";");
    }

    private void readHeader(final int expectedMatchNumber, final List<TemplateTokenInfo> allTokens) throws Exception {
        readLine();
        if (currentLine == null) {
            throw new TemplateException("Template file is empty");
        }
        String header = null;
        skipEmptyLinesAndComments();
        if (currentLine != null) {
            header = currentLine;
            readLine();
        }
        if (header == null) {
            throw new TemplateUserError("Template file has no data");
        }
        if (!header.contains(ORGANISM_NAME_TOKEN) &&
            !header.contains(ORGANISM_TOKEN)) {
            throw new TemplateUserError("Template file has no header line");
        }
        header = header.replaceAll("\"", "");//remove all speech marks - open office puts these in
        final String[] headerTokens = header.split(CSVWriter.UPLOAD_DELIMITER);
        final List<String> recognizedKeys = new ArrayList<>();
        headerKeys = new ArrayList<>();
{'fraction_non_alphanumeric': 0.10002794076557697, 'fraction_numerical': 0.00298034832821086, 'mean_word_length': 2.716857043959848, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3013865', 'n_tokens_mistral': 2728, 'n_tokens_neox': 2684, 'n_words': 698}