<commit_before>package com.orientechnologies.orient.etl.extractor;

import com.orientechnologies.orient.core.command.OCommandContext;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.etl.OETLProcessor;
import com.orientechnologies.orient.etl.OExtractedItem;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import sun.misc.FloatConsts;

import java.io.IOException;
import java.io.Reader;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Created by frank on 10/5/15.
 */
public class OCSVExtractor extends OAbstractSourceExtractor {

  private static String       NULL_STRING     = "NULL";
  protected OExtractedItem    next;
  private Map<String, OType>  columnTypes     = new HashMap<String, OType>();
  private long                skipFrom        = -1;
  private long                skipTo          = -1;
  private Character           stringCharacter = '"';
  private boolean             unicode         = true;
  private Iterator<CSVRecord> recordIterator;
  private CSVFormat           csvFormat;
  private String              nullValue       = NULL_STRING;
  private String              dateFormat      = "yyyy-MM-dd";

  @Override
  public ODocument getConfiguration() {
    return new ODocument()
        .fromJSON("{parameters:["
            + getCommonConfigurationParameters()
            + ",{separator:{optional:true,description:'Column separator'}},"
            + "{columnsOnFirstLine:{optional:true,description:'Columns are described in the first line'}},"
            + "{columns:{optional:true,description:'Columns array containing names, and optionally type after : (e.g.: name:String, age:int'}},"
            + "{nullValue:{optional:true,description:'Value to consider as NULL_STRING. Default is NULL'}},"
            + "{dateFormat:{optional:true,description:'Date format used to parde dates. Default is yyyy-MM-dd'}},"
            + "{quote:{optional:true,description:'String character delimiter. Use \"\" to do not use any delimitator'}},"
            + "{ignoreEmptyLines:{optional:true,description:'Ignore empty lines',type:'boolean'}},"
            + "{skipFrom:{optional:true,description:'Line number where start to skip',type:'int'}},"
            + "{skipTo:{optional:true,description:'Line number where skip ends',type:'int'}}"
            + "{predefinedFormat:{optional:true,description:'Name of standard csv format (from Apache commons-csv): DEFAULT, EXCEL, MYSQL, RFC4180, TDF',type:'String'}}"
            + "],input:['String'],output:'ODocument'}");
  }

  @Override
  public String getName() {
    return "csv";
  }

  @Override
  public String getUnit() {
    return "rows";
  }

  @Override
  public void configure(OETLProcessor iProcessor, ODocument iConfiguration, OCommandContext iContext) {
    super.configure(iProcessor, iConfiguration, iContext);

    csvFormat = CSVFormat.newFormat(',').withNullString(NULL_STRING).withEscape('\\').withQuote('"');

    if (iConfiguration.containsField("predefinedFormat")) {
      csvFormat = CSVFormat.valueOf(iConfiguration.<String> field("predefinedFormat").toUpperCase());
    }

    if (iConfiguration.containsField("separator")) {
      csvFormat = csvFormat.withDelimiter(iConfiguration.field("separator").toString().charAt(0));
    }

    if (iConfiguration.containsField("dateFormat")) {
      dateFormat = iConfiguration.<String> field("dateFormat");
    }

    if (iConfiguration.containsField("ignoreEmptyLines")) {
      boolean ignoreEmptyLines = iConfiguration.field("ignoreEmptyLines");
      csvFormat = csvFormat.withIgnoreEmptyLines(ignoreEmptyLines);
    }

    if (iConfiguration.containsField("columnsOnFirstLine")) {
      boolean columnsOnFirstLine = (Boolean) iConfiguration.field("columnsOnFirstLine");
      if (columnsOnFirstLine == true) {
        csvFormat = csvFormat.withHeader();
      }
    } else {
      csvFormat = csvFormat.withHeader();
    }

    if (iConfiguration.containsField("columns")) {
      final List<String> columns = iConfiguration.field("columns");

      ArrayList<String> columnNames = new ArrayList<String>(columns.size());
      columnTypes = new HashMap<String, OType>(columns.size());
      for (String c : columns) {
        final String[] parts = c.split(":");
        columnNames.add(parts[0]);
        if (parts.length > 1) {
          columnTypes.put(parts[0], OType.valueOf(parts[1].toUpperCase()));
        } else {
          columnTypes.put(parts[0], OType.ANY);
        }
      }

      log(OETLProcessor.LOG_LEVELS.INFO, "column types: %s", columnTypes);
      String[] header = columnTypes.keySet().toArray(new String[] {});
      csvFormat = csvFormat.withHeader(header);

    }
    if (iConfiguration.containsField("skipFrom")) {
      skipFrom = ((Number) iConfiguration.field("skipFrom")).longValue();

    }
    if (iConfiguration.containsField("skipTo")) {
      skipTo = ((Number) iConfiguration.field("skipTo")).longValue();
    }

    if (iConfiguration.containsField("nullValue")) {
      nullValue = iConfiguration.<String> field("nullValue");
      csvFormat.withNullString(nullValue);
    }

    if (iConfiguration.containsField("quote")) {
      final String value = iConfiguration.field("quote").toString();
      if (!value.isEmpty()) {
        csvFormat = csvFormat.withQuote(value.charAt(0));
      }
    }

  }

  @Override
  public void extract(final Reader iReader) {
    super.extract(iReader);
    try {

      CSVParser parser = new CSVParser(iReader, csvFormat);
<commit_msg>fixup wrong mapping of colums when types are declared<commit_after>package com.orientechnologies.orient.etl.extractor;

import com.orientechnologies.orient.core.command.OCommandContext;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.etl.OETLProcessor;
import com.orientechnologies.orient.etl.OExtractedItem;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import sun.misc.FloatConsts;

import java.io.IOException;
import java.io.Reader;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Created by frank on 10/5/15.
 */
public class OCSVExtractor extends OAbstractSourceExtractor {

  private static String       NULL_STRING     = "NULL";
  protected OExtractedItem    next;
  private Map<String, OType>  columnTypes     = new HashMap<String, OType>();
  private long                skipFrom        = -1;
  private long                skipTo          = -1;
  private Character           stringCharacter = '"';
  private boolean             unicode         = true;
  private Iterator<CSVRecord> recordIterator;
  private CSVFormat           csvFormat;
  private String              nullValue       = NULL_STRING;
  private String              dateFormat      = "yyyy-MM-dd";

  @Override
  public ODocument getConfiguration() {
    return new ODocument()
        .fromJSON("{parameters:["
            + getCommonConfigurationParameters()
            + ",{separator:{optional:true,description:'Column separator'}},"
            + "{columnsOnFirstLine:{optional:true,description:'Columns are described in the first line'}},"
            + "{columns:{optional:true,description:'Columns array containing names, and optionally type after : (e.g.: name:String, age:int'}},"
            + "{nullValue:{optional:true,description:'Value to consider as NULL_STRING. Default is NULL'}},"
            + "{dateFormat:{optional:true,description:'Date format used to parde dates. Default is yyyy-MM-dd'}},"
            + "{quote:{optional:true,description:'String character delimiter. Use \"\" to do not use any delimitator'}},"
            + "{ignoreEmptyLines:{optional:true,description:'Ignore empty lines',type:'boolean'}},"
            + "{skipFrom:{optional:true,description:'Line number where start to skip',type:'int'}},"
            + "{skipTo:{optional:true,description:'Line number where skip ends',type:'int'}}"
            + "{predefinedFormat:{optional:true,description:'Name of standard csv format (from Apache commons-csv): DEFAULT, EXCEL, MYSQL, RFC4180, TDF',type:'String'}}"
            + "],input:['String'],output:'ODocument'}");
  }

  @Override
  public String getName() {
    return "csv";
  }

  @Override
  public String getUnit() {
    return "rows";
  }

  @Override
  public void configure(OETLProcessor iProcessor, ODocument iConfiguration, OCommandContext iContext) {
    super.configure(iProcessor, iConfiguration, iContext);

    csvFormat = CSVFormat.newFormat(',').withNullString(NULL_STRING).withEscape('\\').withQuote('"');

    if (iConfiguration.containsField("predefinedFormat")) {
      csvFormat = CSVFormat.valueOf(iConfiguration.<String> field("predefinedFormat").toUpperCase());
    }

    if (iConfiguration.containsField("separator")) {
      csvFormat = csvFormat.withDelimiter(iConfiguration.field("separator").toString().charAt(0));
    }

    if (iConfiguration.containsField("dateFormat")) {
      dateFormat = iConfiguration.<String> field("dateFormat");
    }

    if (iConfiguration.containsField("ignoreEmptyLines")) {
      boolean ignoreEmptyLines = iConfiguration.field("ignoreEmptyLines");
      csvFormat = csvFormat.withIgnoreEmptyLines(ignoreEmptyLines);
    }

    if (iConfiguration.containsField("columnsOnFirstLine")) {
      boolean columnsOnFirstLine = (Boolean) iConfiguration.field("columnsOnFirstLine");
      if (columnsOnFirstLine == true) {
        csvFormat = csvFormat.withHeader();
      }
    } else {
      csvFormat = csvFormat.withHeader();
    }

    if (iConfiguration.containsField("columns")) {
      final List<String> columns = iConfiguration.field("columns");

      ArrayList<String> columnNames = new ArrayList<String>(columns.size());
      columnTypes = new HashMap<String, OType>(columns.size());
      for (String c : columns) {
        final String[] parts = c.split(":");
        columnNames.add(parts[0]);
        if (parts.length > 1) {
          columnTypes.put(parts[0], OType.valueOf(parts[1].toUpperCase()));
        } else {
          columnTypes.put(parts[0], OType.ANY);
        }
      }

      log(OETLProcessor.LOG_LEVELS.INFO, "column types: %s", columnTypes);
      csvFormat = csvFormat.withHeader(columnNames.toArray(new String[] {}));

    }
    if (iConfiguration.containsField("skipFrom")) {
      skipFrom = ((Number) iConfiguration.field("skipFrom")).longValue();

    }
    if (iConfiguration.containsField("skipTo")) {
      skipTo = ((Number) iConfiguration.field("skipTo")).longValue();
    }

    if (iConfiguration.containsField("nullValue")) {
      nullValue = iConfiguration.<String> field("nullValue");
      csvFormat.withNullString(nullValue);
    }

    if (iConfiguration.containsField("quote")) {
      final String value = iConfiguration.field("quote").toString();
      if (!value.isEmpty()) {
        csvFormat = csvFormat.withQuote(value.charAt(0));
      }
    }

  }

  @Override
  public void extract(final Reader iReader) {
    super.extract(iReader);
    try {

      CSVParser parser = new CSVParser(iReader, csvFormat);

{'fraction_non_alphanumeric': 0.13148455953109964, 'fraction_numerical': 0.0031493307672119677, 'mean_word_length': 3.591164658634538, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3319253', 'n_tokens_mistral': 3266, 'n_tokens_neox': 3146, 'n_words': 629}