<commit_before>package org.gbif.occurrence.download.file.dwca;

import org.gbif.api.model.occurrence.Download;
import org.gbif.api.model.occurrence.predicate.Predicate;
import org.gbif.api.model.registry.Citation;
import org.gbif.api.model.registry.Dataset;
import org.gbif.api.model.registry.Identifier;
import org.gbif.api.model.registry.eml.DataDescription;
import org.gbif.api.service.registry.DatasetService;
import org.gbif.api.service.registry.OccurrenceDownloadService;
import org.gbif.api.vocabulary.ContactType;
import org.gbif.api.vocabulary.DatasetType;
import org.gbif.api.vocabulary.IdentifierType;
import org.gbif.api.vocabulary.Language;
import org.gbif.api.vocabulary.License;
import org.gbif.hadoop.compress.d2.D2CombineInputStream;
import org.gbif.hadoop.compress.d2.D2Utils;
import org.gbif.hadoop.compress.d2.zip.ModalZipOutputStream;
import org.gbif.occurrence.common.download.DownloadException;
import org.gbif.occurrence.download.conf.WorkflowConfiguration;
import org.gbif.occurrence.download.file.DownloadJobConfiguration;
import org.gbif.occurrence.download.license.LicenseSelector;
import org.gbif.occurrence.download.license.LicenseSelectors;
import org.gbif.occurrence.download.util.HeadersFileUtil;
import org.gbif.occurrence.download.util.RegistryClientUtil;
import org.gbif.occurrence.query.HumanFilterBuilder;
import org.gbif.occurrence.query.TitleLookup;
import org.gbif.occurrence.query.TitleLookupModule;
import org.gbif.registry.metadata.EMLWriter;
import org.gbif.utils.file.CompressionUtil;
import org.gbif.utils.file.FileUtils;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Writer;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.base.Objects;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.sun.jersey.api.client.UniformInterfaceException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.CITATIONS_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.INTERPRETED_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.METADATA_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.MULTIMEDIA_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.RIGHTS_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.VERBATIM_FILENAME;

/**
 * Creates a dwc archive for occurrence downloads based on the hive query result files generated
 * during the Oozie workflow. It create a local archive folder with an occurrence data file and a dataset subfolder
 * that contains an EML metadata file per dataset involved.
 */
public class DwcaArchiveBuilder {

  private static final Logger LOG = LoggerFactory.getLogger(DwcaArchiveBuilder.class);
  // The CRC is created by the function FileSystem.copyMerge function
  private static final String DOWNLOAD_CONTACT_SERVICE = "GBIF Download Service";
  private static final String DOWNLOAD_CONTACT_EMAIL = "<EMAIL>";
  private static final String METADATA_DESC_HEADER_FMT =
    "A dataset containing all occurrences available in GBIF matching the query:\n%s"
    +
    "\nThe dataset includes records from the following constituent datasets. "
    + "The full metadata for each constituent is also included in this archive:\n";
  private static final String CITATION_HEADER =
    "When using this dataset please use the following citation and pay attention to the rights documented in the rights.txt:";
  private static final String DATASET_TITLE_FMT = "GBIF Occurrence Download %s";
  private static final String RIGHTS =
    "The data included in this download are provided to the user under a %s license (%s), please read the license terms and conditions to understand the implications of its usage and sharing.\nData from some individual datasets included in this download may be licensed under less restrictive terms; review the details below.";
  private static final String DATA_DESC_FORMAT = "Darwin Core Archive";
  private static final Splitter TAB_SPLITTER = Splitter.on('\t').trimResults();
  private static final EMLWriter EML_WRITER = EMLWriter.newInstance(true);

  private final DatasetService datasetService;

  private final OccurrenceDownloadService occurrenceDownloadService;
  private final TitleLookup titleLookup;
  private final File archiveDir;
  private final WorkflowConfiguration workflowConfiguration;
  private final FileSystem sourceFs;
  private final FileSystem targetFs;
  private final DownloadJobConfiguration configuration;
  private final LicenseSelector licenseSelector = LicenseSelectors.getMostRestrictiveLicenseSelector(License.CC_BY_4_0);

  //constituents and citation are basically the same info, are keep in 2 separate collections to avoid rebuilding them
  private Set<Constituent> constituents = Sets.newTreeSet();
  private Map<UUID,Long> citations = Maps.newHashMap();

  public static void buildArchive(DownloadJobConfiguration configuration) throws IOException {
    buildArchive(configuration, new WorkflowConfiguration());
  }

  public static void buildArchive(DownloadJobConfiguration configuration, WorkflowConfiguration workflowConfiguration)
    throws IOException {
    RegistryClientUtil registryClientUtil = new RegistryClientUtil();
    String tmpDir = workflowConfiguration.getTempDir();

    // create temporary, local, download specific directory
    File archiveDir = new File(tmpDir, configuration.getDownloadKey());

    String registryWs = workflowConfiguration.getRegistryWsUrl();
    // create registry client and services
    DatasetService datasetService = registryClientUtil.setupDatasetService(registryWs);
    OccurrenceDownloadService occurrenceDownloadService = registryClientUtil.setupOccurrenceDownloadService(registryWs);

    Injector inj = Guice.createInjector(new TitleLookupModule(true, workflowConfiguration.getApiUrl()));
    TitleLookup titleLookup = inj.getInstance(TitleLookup.class);

    FileSystem sourceFs = configuration.isSmallDownload()
      ? FileSystem.getLocal(workflowConfiguration.getHadoopConf())
      : FileSystem.get(workflowConfiguration.getHadoopConf());
    FileSystem targetFs = FileSystem.get(workflowConfiguration.getHadoopConf());

    // build archive
    DwcaArchiveBuilder generator = new DwcaArchiveBuilder(datasetService,
                                                          occurrenceDownloadService,
                                                          sourceFs,
                                                          targetFs,
                                                          archiveDir,
                                                          titleLookup,
                                                          configuration,
                                                          workflowConfiguration);
    generator.buildArchive(new File(tmpDir, configuration.getDownloadKey() + ".zip"));
  }

  private static String writeCitation(Writer citationWriter, Dataset dataset)
    throws IOException {
    // citation
    String citationLink = null;
    if (dataset.getCitation() != null && !Strings.isNullOrEmpty(dataset.getCitation().getText())) {
      citationWriter.write('\n' + dataset.getCitation().getText());
      if (!Strings.isNullOrEmpty(dataset.getCitation().getIdentifier())) {
        citationLink = ", " + dataset.getCitation().getIdentifier();
        citationWriter.write(citationLink);
      }
    } else {
      LOG.error("Constituent dataset misses mandatory citation for id: {}", dataset.getKey());
    }
    if (dataset.getDoi() != null) {
      citationWriter.write(" " + dataset.getDoi());
    }
    return citationLink;
  }

  /**
   * Write rights text.
   */
  private static void writeRights(Writer rightsWriter, Dataset dataset, String citationLink)
    throws IOException {
    // write rights
    rightsWriter.write("\nDataset: " + dataset.getTitle());<commit_msg>Remove superfluous DOIs in download citations.txt.<commit_after>package org.gbif.occurrence.download.file.dwca;

import org.gbif.api.model.occurrence.Download;
import org.gbif.api.model.occurrence.predicate.Predicate;
import org.gbif.api.model.registry.Citation;
import org.gbif.api.model.registry.Dataset;
import org.gbif.api.model.registry.Identifier;
import org.gbif.api.model.registry.eml.DataDescription;
import org.gbif.api.service.registry.DatasetService;
import org.gbif.api.service.registry.OccurrenceDownloadService;
import org.gbif.api.vocabulary.ContactType;
import org.gbif.api.vocabulary.DatasetType;
import org.gbif.api.vocabulary.IdentifierType;
import org.gbif.api.vocabulary.Language;
import org.gbif.api.vocabulary.License;
import org.gbif.hadoop.compress.d2.D2CombineInputStream;
import org.gbif.hadoop.compress.d2.D2Utils;
import org.gbif.hadoop.compress.d2.zip.ModalZipOutputStream;
import org.gbif.occurrence.common.download.DownloadException;
import org.gbif.occurrence.download.conf.WorkflowConfiguration;
import org.gbif.occurrence.download.file.DownloadJobConfiguration;
import org.gbif.occurrence.download.license.LicenseSelector;
import org.gbif.occurrence.download.license.LicenseSelectors;
import org.gbif.occurrence.download.util.HeadersFileUtil;
import org.gbif.occurrence.download.util.RegistryClientUtil;
import org.gbif.occurrence.query.HumanFilterBuilder;
import org.gbif.occurrence.query.TitleLookup;
import org.gbif.occurrence.query.TitleLookupModule;
import org.gbif.registry.metadata.EMLWriter;
import org.gbif.utils.file.CompressionUtil;
import org.gbif.utils.file.FileUtils;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Writer;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.base.Objects;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.sun.jersey.api.client.UniformInterfaceException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.CITATIONS_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.INTERPRETED_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.METADATA_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.MULTIMEDIA_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.RIGHTS_FILENAME;
import static org.gbif.occurrence.download.file.dwca.DwcDownloadsConstants.VERBATIM_FILENAME;

/**
 * Creates a dwc archive for occurrence downloads based on the hive query result files generated
 * during the Oozie workflow. It create a local archive folder with an occurrence data file and a dataset subfolder
 * that contains an EML metadata file per dataset involved.
 */
public class DwcaArchiveBuilder {

  private static final Logger LOG = LoggerFactory.getLogger(DwcaArchiveBuilder.class);
  // The CRC is created by the function FileSystem.copyMerge function
  private static final String DOWNLOAD_CONTACT_SERVICE = "GBIF Download Service";
  private static final String DOWNLOAD_CONTACT_EMAIL = "<EMAIL>";
  private static final String METADATA_DESC_HEADER_FMT =
    "A dataset containing all occurrences available in GBIF matching the query:\n%s"
    +
    "\nThe dataset includes records from the following constituent datasets. "
    + "The full metadata for each constituent is also included in this archive:\n";
  private static final String CITATION_HEADER =
    "When using this dataset please use the following citation and pay attention to the rights documented in rights.txt:\n";
  private static final String DATASET_TITLE_FMT = "GBIF Occurrence Download %s";
  private static final String RIGHTS =
    "The data included in this download are provided to the user under a %s license (%s), please read the license terms and conditions to understand the implications of its usage and sharing.\nData from some individual datasets included in this download may be licensed under less restrictive terms; review the details below.";
  private static final String DATA_DESC_FORMAT = "Darwin Core Archive";
  private static final Splitter TAB_SPLITTER = Splitter.on('\t').trimResults();
  private static final EMLWriter EML_WRITER = EMLWriter.newInstance(true);

  private final DatasetService datasetService;

  private final OccurrenceDownloadService occurrenceDownloadService;
  private final TitleLookup titleLookup;
  private final File archiveDir;
  private final WorkflowConfiguration workflowConfiguration;
  private final FileSystem sourceFs;
  private final FileSystem targetFs;
  private final DownloadJobConfiguration configuration;
  private final LicenseSelector licenseSelector = LicenseSelectors.getMostRestrictiveLicenseSelector(License.CC_BY_4_0);

  //constituents and citation are basically the same info, are keep in 2 separate collections to avoid rebuilding them
  private Set<Constituent> constituents = Sets.newTreeSet();
  private Map<UUID,Long> citations = Maps.newHashMap();

  public static void buildArchive(DownloadJobConfiguration configuration) throws IOException {
    buildArchive(configuration, new WorkflowConfiguration());
  }

  public static void buildArchive(DownloadJobConfiguration configuration, WorkflowConfiguration workflowConfiguration)
    throws IOException {
    RegistryClientUtil registryClientUtil = new RegistryClientUtil();
    String tmpDir = workflowConfiguration.getTempDir();

    // create temporary, local, download specific directory
    File archiveDir = new File(tmpDir, configuration.getDownloadKey());

    String registryWs = workflowConfiguration.getRegistryWsUrl();
    // create registry client and services
    DatasetService datasetService = registryClientUtil.setupDatasetService(registryWs);
    OccurrenceDownloadService occurrenceDownloadService = registryClientUtil.setupOccurrenceDownloadService(registryWs);

    Injector inj = Guice.createInjector(new TitleLookupModule(true, workflowConfiguration.getApiUrl()));
    TitleLookup titleLookup = inj.getInstance(TitleLookup.class);

    FileSystem sourceFs = configuration.isSmallDownload()
      ? FileSystem.getLocal(workflowConfiguration.getHadoopConf())
      : FileSystem.get(workflowConfiguration.getHadoopConf());
    FileSystem targetFs = FileSystem.get(workflowConfiguration.getHadoopConf());

    // build archive
    DwcaArchiveBuilder generator = new DwcaArchiveBuilder(datasetService,
                                                          occurrenceDownloadService,
                                                          sourceFs,
                                                          targetFs,
                                                          archiveDir,
                                                          titleLookup,
                                                          configuration,
                                                          workflowConfiguration);
    generator.buildArchive(new File(tmpDir, configuration.getDownloadKey() + ".zip"));
  }

  private static String writeCitation(Writer citationWriter, Dataset dataset)
    throws IOException {
    // citation
    String citationLink = null;
    if (dataset.getCitation() != null && !Strings.isNullOrEmpty(dataset.getCitation().getText())) {
      citationWriter.write(dataset.getCitation().getText());
      citationWriter.write('\n');
    } else {
      LOG.error("Constituent dataset misses mandatory citation for id: {}", dataset.getKey());
    }
    return citationLink;
  }

  /**
   * Write rights text.
   */
  private static void writeRights(Writer rightsWriter, Dataset dataset, String citationLink)
    throws IOException {
    // write rights
    rightsWriter.write("\nDataset: " + dataset.getTitle());
{'fraction_non_alphanumeric': 0.08266888150609081, 'fraction_numerical': 0.0011074197120708748, 'mean_word_length': 4.27482476635514, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 113, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '685008', 'n_tokens_mistral': 4906, 'n_tokens_neox': 4701, 'n_words': 1169}