<commit_before>package org.agmip.translators.csv;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.zip.ZipFile;
import java.util.zip.ZipEntry;

import au.com.bytecode.opencsv.CSVReader;
import java.util.HashSet;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.agmip.ace.AcePathfinder;
import org.agmip.ace.util.AcePathfinderUtil;
import org.agmip.core.types.TranslatorInput;

/**
 * This class converts CSV formatted files into the AgMIP ACE JSON format. It
 * uses a common file pattern as described below.
 *
 * <p><b>First Column Descriptors</b></p> <p># - Lines with the first column
 * text containing only a "#" is considered a header row</p> <p>! - Lines with
 * the first column text containing only a "!" are considered a comment and not
 * parsed.
 *
 * The first header/datarow(s) are metadata (or global data) if there are
 * multiple rows of metadata, they are considered to be a collection of
 * experiments.
 *
 */
public class CSVInput implements TranslatorInput {

    private static Logger LOG = LoggerFactory.getLogger(CSVInput.class);
//    private HashMap<String, HashMap<String, HashMap<String, Object>>> finalMap;
    private HashMap<String, HashMap<String, Object>> expMap, weatherMap, soilMap; // Storage maps
    private HashMap<String, Integer> trtTracker;
    private HashMap<String, String> idMap;
    private ArrayList<String> orderring;
    private String listSeparator;
    private AcePathfinder pathfinder = AcePathfinderUtil.getInstance();
    private static HashSet unknowVars = new HashSet();

    private enum HeaderType {

        UNKNOWN, // Probably uninitialized
        SUMMARY, // #
        SERIES   // %
    }

    private static class CSVHeader {

        private final ArrayList<String> headers;
        private final ArrayList<Integer> skippedColumns;
        private final String defPath;
        private final AcePathfinderUtil.PathType defPathType;

        public CSVHeader(ArrayList<String> headers, ArrayList<Integer> sc) {
            this(headers, sc, null, AcePathfinderUtil.PathType.UNKNOWN);
        }

        public CSVHeader(ArrayList<String> headers, ArrayList<Integer> sc, String defPath, AcePathfinderUtil.PathType defPathType) {
            this.headers = headers;
            this.skippedColumns = sc;
            this.defPath = defPath;
            this.defPathType = defPathType;
        }

        public CSVHeader() {
            this.headers = new ArrayList<String>();
            this.skippedColumns = new ArrayList<Integer>();
            this.defPath = null;
            this.defPathType = AcePathfinderUtil.PathType.UNKNOWN;
        }

        public ArrayList<String> getHeaders() {
            return headers;
        }

        public ArrayList<Integer> getSkippedColumns() {
            return skippedColumns;
        }

        public String getDefPath() {
            return defPath;
        }

        public AcePathfinderUtil.PathType getDefPathType() {
            return defPathType;
        }
    }

    public CSVInput() {
        expMap = new HashMap<String, HashMap<String, Object>>();
        weatherMap = new HashMap<String, HashMap<String, Object>>();
        soilMap = new HashMap<String, HashMap<String, Object>>();
        trtTracker = new HashMap<String, Integer>();
        idMap = new HashMap<String, String>();

        orderring = new ArrayList<String>();
//        finalMap = new HashMap<String, HashMap<String, HashMap<String, Object>>>();
        this.listSeparator = ",";
//        finalMap.put("experiments", expMap);
//        finalMap.put("weather", weatherMap);
//        finalMap.put("soil", soilMap);
    }

    @Override
    public Map readFile(String fileName) throws Exception {
        if (fileName.toUpperCase().endsWith("CSV")) {
            readCSV(new FileInputStream(fileName));
        } else if (fileName.toUpperCase().endsWith("ZIP")) {
            //Handle a ZipInputStream instead
            LOG.debug("Launching zip file handler");
            ZipFile zf = new ZipFile(fileName);
            Enumeration<? extends ZipEntry> e = zf.entries();
            while (e.hasMoreElements()) {
                ZipEntry ze = (ZipEntry) e.nextElement();
                LOG.debug("Entering file: " + ze);
                readCSV(zf.getInputStream(ze));
            }
            zf.close();
        }
        return cleanUpFinalMap();
    }

    protected void readCSV(InputStream fileStream) throws Exception {
        HeaderType section = HeaderType.UNKNOWN;
        CSVHeader currentHeader = new CSVHeader();
        String[] nextLine;
        BufferedReader br = new BufferedReader(new InputStreamReader(fileStream));

        // Check to see if this is an international CSV. (;, vs ,.)
        setListSeparator(br);
        CSVReader reader = new CSVReader(br, this.listSeparator.charAt(0));
<commit_msg>Add file extension filter to only read CSV file<commit_after>package org.agmip.translators.csv;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.zip.ZipFile;
import java.util.zip.ZipEntry;

import au.com.bytecode.opencsv.CSVReader;
import java.util.HashSet;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.agmip.ace.AcePathfinder;
import org.agmip.ace.util.AcePathfinderUtil;
import org.agmip.core.types.TranslatorInput;

/**
 * This class converts CSV formatted files into the AgMIP ACE JSON format. It
 * uses a common file pattern as described below.
 *
 * <p><b>First Column Descriptors</b></p> <p># - Lines with the first column
 * text containing only a "#" is considered a header row</p> <p>! - Lines with
 * the first column text containing only a "!" are considered a comment and not
 * parsed.
 *
 * The first header/datarow(s) are metadata (or global data) if there are
 * multiple rows of metadata, they are considered to be a collection of
 * experiments.
 *
 */
public class CSVInput implements TranslatorInput {

    private static Logger LOG = LoggerFactory.getLogger(CSVInput.class);
//    private HashMap<String, HashMap<String, HashMap<String, Object>>> finalMap;
    private HashMap<String, HashMap<String, Object>> expMap, weatherMap, soilMap; // Storage maps
    private HashMap<String, Integer> trtTracker;
    private HashMap<String, String> idMap;
    private ArrayList<String> orderring;
    private String listSeparator;
    private AcePathfinder pathfinder = AcePathfinderUtil.getInstance();
    private static HashSet unknowVars = new HashSet();

    private enum HeaderType {

        UNKNOWN, // Probably uninitialized
        SUMMARY, // #
        SERIES   // %
    }

    private static class CSVHeader {

        private final ArrayList<String> headers;
        private final ArrayList<Integer> skippedColumns;
        private final String defPath;
        private final AcePathfinderUtil.PathType defPathType;

        public CSVHeader(ArrayList<String> headers, ArrayList<Integer> sc) {
            this(headers, sc, null, AcePathfinderUtil.PathType.UNKNOWN);
        }

        public CSVHeader(ArrayList<String> headers, ArrayList<Integer> sc, String defPath, AcePathfinderUtil.PathType defPathType) {
            this.headers = headers;
            this.skippedColumns = sc;
            this.defPath = defPath;
            this.defPathType = defPathType;
        }

        public CSVHeader() {
            this.headers = new ArrayList<String>();
            this.skippedColumns = new ArrayList<Integer>();
            this.defPath = null;
            this.defPathType = AcePathfinderUtil.PathType.UNKNOWN;
        }

        public ArrayList<String> getHeaders() {
            return headers;
        }

        public ArrayList<Integer> getSkippedColumns() {
            return skippedColumns;
        }

        public String getDefPath() {
            return defPath;
        }

        public AcePathfinderUtil.PathType getDefPathType() {
            return defPathType;
        }
    }

    public CSVInput() {
        expMap = new HashMap<String, HashMap<String, Object>>();
        weatherMap = new HashMap<String, HashMap<String, Object>>();
        soilMap = new HashMap<String, HashMap<String, Object>>();
        trtTracker = new HashMap<String, Integer>();
        idMap = new HashMap<String, String>();

        orderring = new ArrayList<String>();
//        finalMap = new HashMap<String, HashMap<String, HashMap<String, Object>>>();
        this.listSeparator = ",";
//        finalMap.put("experiments", expMap);
//        finalMap.put("weather", weatherMap);
//        finalMap.put("soil", soilMap);
    }

    @Override
    public Map readFile(String fileName) throws Exception {
        if (fileName.toUpperCase().endsWith("CSV")) {
            readCSV(new FileInputStream(fileName));
        } else if (fileName.toUpperCase().endsWith("ZIP")) {
            //Handle a ZipInputStream instead
            LOG.debug("Launching zip file handler");
            ZipFile zf = new ZipFile(fileName);
            Enumeration<? extends ZipEntry> e = zf.entries();
            while (e.hasMoreElements()) {
                ZipEntry ze = (ZipEntry) e.nextElement();
                LOG.debug("Entering file: " + ze);
                if (ze.getName().toLowerCase().endsWith(".csv")) {
                    readCSV(zf.getInputStream(ze));
                }
            }
            zf.close();
        }
        return cleanUpFinalMap();
    }

    protected void readCSV(InputStream fileStream) throws Exception {
        HeaderType section = HeaderType.UNKNOWN;
        CSVHeader currentHeader = new CSVHeader();
        String[] nextLine;
        BufferedReader br = new BufferedReader(new InputStreamReader(fileStream));

        // Check to see if this is an international CSV. (;, vs ,.)
        setListSeparator(br);
        CSVReader reader = new CSVReader(br, this.listSeparator.charAt(0));

{'fraction_non_alphanumeric': 0.10313428544042941, 'fraction_numerical': 0.0005750982459503498, 'mean_word_length': 2.792802617230098, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2009295', 'n_tokens_mistral': 2903, 'n_tokens_neox': 2809, 'n_words': 770}