<commit_before>/*
 * Copyright 2010 Global Biodiversity Informatics Facility.
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package org.gbif.dwc;

import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
import org.gbif.dwc.terms.Term;
import org.gbif.dwc.terms.TermFactory;
import org.gbif.dwc.record.Record;
import org.gbif.utils.file.ClosableIterator;
import org.gbif.utils.file.FileUtils;
import org.gbif.utils.file.tabular.TabularDataFileReader;
import org.gbif.utils.file.tabular.TabularFileNormalizer;
import org.gbif.utils.file.tabular.TabularFiles;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.stream.Collectors;

/**
 * This class encapsulates information about a file contained within a Darwin Core Archive. It
 * represents the fileType object described in the Darwin Core Archive XSD.
 *
 * @see <a href="http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd">Darwin Core Archive XSD</a>
 */
public class ArchiveFile implements Iterable<Record> {
  private static final Logger LOG = LoggerFactory.getLogger(ArchiveFile.class);

  private static final FileUtils FILE_UTILS = new FileUtils();
  private static final TermFactory TERM_FACTORY = TermFactory.instance();

  public static final Term DEFAULT_ID_TERM = TERM_FACTORY.findPropertyTerm("ARCHIVE_RECORD_ID");
  public static final Character DEFAULT_FIELDS_ENCLOSED_BY = '"';
  public static final String DEFAULT_FIELDS_TERMINATED_BY = ",";
  public static final String DEFAULT_LINES_TERMINATED_BY = "\n";

  private static final Comparator<ArchiveField> AF_IDX_COMPARATOR = (o1, o2) -> {
    if (o1.getIndex() == null && o2.getIndex() == null) {
      return 0;
    } else if (o1.getIndex() == null) {
      return -1;
    } else if (o2.getIndex() == null) {
      return 1;
    } else if (o1.getIndex().equals(o2.getIndex())) {
      return o1.getTerm().qualifiedName().compareToIgnoreCase(o2.getTerm().qualifiedName());
    } else {
      return o1.getIndex().compareTo(o2.getIndex());
    }
  };

  private ArchiveField id;
  private Archive archive;
  private final LinkedList<String> locations = new LinkedList<String>();
  private String title;
  private String fieldsTerminatedBy = DEFAULT_FIELDS_TERMINATED_BY;
  private Character fieldsEnclosedBy = DEFAULT_FIELDS_ENCLOSED_BY;
  private String linesTerminatedBy = DEFAULT_LINES_TERMINATED_BY;
  private String encoding = FileUtils.UTF8;
  private Term rowType; // Default is http://rs.tdwg.org/dwc/xsd/simpledarwincore/SimpleDarwinRecord
  private Integer ignoreHeaderLines = 0;
  private String dateFormat = "YYYY-MM-DD";

  // TODO: Change to SortedMap and remove rawArchiveFields?
  private final Map<Term, ArchiveField> fields = new HashMap<>();
  private final List<ArchiveField> rawArchiveFields = new ArrayList<>();

  public static ArchiveFile buildCsvFile() {
    ArchiveFile af = new ArchiveFile();
    af.setFieldsEnclosedBy(DEFAULT_FIELDS_ENCLOSED_BY);
    af.setFieldsTerminatedBy(DEFAULT_FIELDS_TERMINATED_BY);
    return af;
  }
<commit_msg>Case-insensitive sort so it won't depend on locale.<commit_after>/*
 * Copyright 2010 Global Biodiversity Informatics Facility.
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package org.gbif.dwc;

import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
import org.gbif.dwc.terms.Term;
import org.gbif.dwc.terms.TermFactory;
import org.gbif.dwc.record.Record;
import org.gbif.utils.file.ClosableIterator;
import org.gbif.utils.file.FileUtils;
import org.gbif.utils.file.tabular.TabularDataFileReader;
import org.gbif.utils.file.tabular.TabularFileNormalizer;
import org.gbif.utils.file.tabular.TabularFiles;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.stream.Collectors;

/**
 * This class encapsulates information about a file contained within a Darwin Core Archive. It
 * represents the fileType object described in the Darwin Core Archive XSD.
 *
 * @see <a href="http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd">Darwin Core Archive XSD</a>
 */
public class ArchiveFile implements Iterable<Record> {
  private static final Logger LOG = LoggerFactory.getLogger(ArchiveFile.class);

  private static final FileUtils FILE_UTILS = new FileUtils();
  private static final TermFactory TERM_FACTORY = TermFactory.instance();

  public static final Term DEFAULT_ID_TERM = TERM_FACTORY.findPropertyTerm("ARCHIVE_RECORD_ID");
  public static final Character DEFAULT_FIELDS_ENCLOSED_BY = '"';
  public static final String DEFAULT_FIELDS_TERMINATED_BY = ",";
  public static final String DEFAULT_LINES_TERMINATED_BY = "\n";

  /**
   * Comparator to sort by column index, then term name.
   */
  private static final Comparator<ArchiveField> AF_IDX_COMPARATOR = (o1, o2) -> {
    if (o1.getIndex() == null && o2.getIndex() == null) {
      return 0;
    } else if (o1.getIndex() == null) {
      return -1;
    } else if (o2.getIndex() == null) {
      return 1;
    } else if (o1.getIndex().equals(o2.getIndex())) {
      return o1.getTerm().qualifiedName().compareTo(o2.getTerm().qualifiedName());
    } else {
      return o1.getIndex().compareTo(o2.getIndex());
    }
  };

  private ArchiveField id;
  private Archive archive;
  private final LinkedList<String> locations = new LinkedList<String>();
  private String title;
  private String fieldsTerminatedBy = DEFAULT_FIELDS_TERMINATED_BY;
  private Character fieldsEnclosedBy = DEFAULT_FIELDS_ENCLOSED_BY;
  private String linesTerminatedBy = DEFAULT_LINES_TERMINATED_BY;
  private String encoding = FileUtils.UTF8;
  private Term rowType; // Default is http://rs.tdwg.org/dwc/xsd/simpledarwincore/SimpleDarwinRecord
  private Integer ignoreHeaderLines = 0;
  private String dateFormat = "YYYY-MM-DD";

  // TODO: Change to SortedMap and remove rawArchiveFields?
  private final Map<Term, ArchiveField> fields = new HashMap<>();
  private final List<ArchiveField> rawArchiveFields = new ArrayList<>();

  public static ArchiveFile buildCsvFile() {
    ArchiveFile af = new ArchiveFile();
    af.setFieldsEnclosedBy(DEFAULT_FIELDS_ENCLOSED_BY);
    af.setFieldsTerminatedBy(DEFAULT_FIELDS_TERMINATED_BY);
    return af;
  }

{'fraction_non_alphanumeric': 0.09935176610662787, 'fraction_numerical': 0.007408387352824447, 'mean_word_length': 4.620817843866171, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '664054', 'n_tokens_mistral': 2366, 'n_tokens_neox': 2282, 'n_words': 646}