<commit_before>package com.metaweb.gridworks.importers;

import java.io.InputStream;
import java.io.LineNumberReader;
import java.io.Reader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang.StringUtils;

import com.metaweb.gridworks.importers.parsers.CSVRowParser;
import com.metaweb.gridworks.importers.parsers.RowParser;
import com.metaweb.gridworks.importers.parsers.SeparatorRowParser;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;

public class TsvCsvImporter implements Importer {

    public void read(Reader reader, Project project, Properties options) throws Exception {
        int limit = ImporterUtilities.getIntegerOption("limit",options,-1);
        int skip = ImporterUtilities.getIntegerOption("skip",options,0);
        boolean guessValueType = ImporterUtilities.getBooleanOption("guess-value-type", options, true);
                
        LineNumberReader lnReader = new LineNumberReader(reader);
        String      sep = options.getProperty("separator"); // auto-detect if not present
        String      line = null;
        boolean     first = true;
        int         cellCount = 1;
        RowParser   parser = (sep == null || (sep.length() == 0)) ? null : new SeparatorRowParser(sep);
        
        int rowsWithData = 0;
        while ((line = lnReader.readLine()) != null) {
            if (StringUtils.isBlank(line)) {
                continue;
            }
            
            if (parser == null) {
                int tab = line.indexOf('\t');
                if (tab >= 0) {
                    sep = "\t";
                    parser = new SeparatorRowParser(sep);
                } else {
                    sep = ",";
                    parser = new CSVRowParser();
                }
            }
            
            if (first) {
                List<String> cells = parser.split(line);
                Map<String, Integer> nameToIndex = new HashMap<String, Integer>();
                                    
                first = false;
                for (int c = 0; c < cells.size(); c++) {
                    String cell = cells.get(c);
                    if (cell.startsWith("\"") && cell.endsWith("\"")) {
                        cell = cell.substring(1, cell.length() - 1);
                    }
                    <commit_msg>Trim column names on import.<commit_after>package com.metaweb.gridworks.importers;

import java.io.InputStream;
import java.io.LineNumberReader;
import java.io.Reader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang.StringUtils;

import com.metaweb.gridworks.importers.parsers.CSVRowParser;
import com.metaweb.gridworks.importers.parsers.RowParser;
import com.metaweb.gridworks.importers.parsers.SeparatorRowParser;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;

public class TsvCsvImporter implements Importer {

    public void read(Reader reader, Project project, Properties options) throws Exception {
        int limit = ImporterUtilities.getIntegerOption("limit",options,-1);
        int skip = ImporterUtilities.getIntegerOption("skip",options,0);
        boolean guessValueType = ImporterUtilities.getBooleanOption("guess-value-type", options, true);
                
        LineNumberReader lnReader = new LineNumberReader(reader);
        String      sep = options.getProperty("separator"); // auto-detect if not present
        String      line = null;
        boolean     first = true;
        int         cellCount = 1;
        RowParser   parser = (sep == null || (sep.length() == 0)) ? null : new SeparatorRowParser(sep);
        
        int rowsWithData = 0;
        while ((line = lnReader.readLine()) != null) {
            if (StringUtils.isBlank(line)) {
                continue;
            }
            
            if (parser == null) {
                int tab = line.indexOf('\t');
                if (tab >= 0) {
                    sep = "\t";
                    parser = new SeparatorRowParser(sep);
                } else {
                    sep = ",";
                    parser = new CSVRowParser();
                }
            }
            
            if (first) {
                List<String> cells = parser.split(line);
                Map<String, Integer> nameToIndex = new HashMap<String, Integer>();
                                    
                first = false;
                for (int c = 0; c < cells.size(); c++) {
                    String cell = cells.get(c).trim();
                    if (cell.startsWith("\"") && cell.endsWith("\"")) {
                        cell = cell.substring(1, cell.length() - 1).trim();
                    }
                    
{'fraction_non_alphanumeric': 0.1107588666534575, 'fraction_numerical': 0.0035664751337428175, 'mean_word_length': 1.8812785388127853, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6647398', 'n_tokens_mistral': 1368, 'n_tokens_neox': 1318, 'n_words': 296}