<commit_before>package org.col.release;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.common.io.Files;
import freemarker.template.*;
import no.api.freemarker.java8.Java8ObjectWrapper;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.col.WsServerConfig;
import org.col.api.model.Dataset;
import org.col.api.model.Page;
import org.col.api.search.DatasetSearchRequest;
import org.col.common.io.CompressionUtil;
import org.col.common.io.Utf8IOUtils;
import org.col.db.mapper.DatasetMapper;
import org.col.img.ImgConfig;
import org.col.postgres.PgCopyUtils;
import org.gbif.nameparser.api.Rank;
import org.postgresql.jdbc.PgConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class AcExporter {
  private static final Logger LOG = LoggerFactory.getLogger(AcExporter.class);
  private static final String EXPORT_SQL = "/exporter/ac-export.sql";
  private static final String COPY_WITH = "CSV HEADER NULL '\\N'";
  private static final Pattern COPY_START = Pattern.compile("^\\s*COPY\\s*\\(");
  private static final Pattern COPY_END   = Pattern.compile("^\\s*\\)\\s*TO\\s*'(.+)'");
  private static final Pattern VAR_DATASET_KEY = Pattern.compile("\\{\\{datasetKey}}", Pattern.CASE_INSENSITIVE);
  private static final Version freemarkerVersion = Configuration.VERSION_2_3_28;
  private static final Configuration fmk = new Configuration(freemarkerVersion);
  static {
    fmk.setClassForTemplateLoading(AcExporter.class, "/exporter");
    // see https://freemarker.apache.org/docs/pgui_quickstart_createconfiguration.html
    fmk.setDefaultEncoding("UTF-8");
    fmk.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
    fmk.setLogTemplateExceptions(false);
    fmk.setWrapUncheckedExceptions(true);
    // allow the use of java8 dates, see https://github.com/lazee/freemarker-java-8
    fmk.setObjectWrapper(new Java8ObjectWrapper(freemarkerVersion));
  }
  private final WsServerConfig cfg;
  private final SqlSessionFactory factory;
  private org.col.release.Logger logger;

  public AcExporter(WsServerConfig cfg, SqlSessionFactory factory) {
    this.cfg = cfg;
    this.factory = factory;
  }
  
  /**
   * @return final archive
   */
  public File export(int catalogueKey, org.col.release.Logger logger) throws IOException, SQLException, IllegalStateException {
    File expDir = new File(cfg.normalizer.scratchDir(catalogueKey), "exports");
    LOG.info("Export catalogue {} to {}", catalogueKey, expDir.getAbsolutePath());
    try {
      logger.setLog(LOG);
      this.logger = logger;
      // create csv files
      try (Connection c = cfg.db.connect()) {
        c.setAutoCommit(false);
        setupTables(c);
        InputStream sql = AcExporter.class.getResourceAsStream(EXPORT_SQL);
        executeAcExportSql(catalogueKey, (PgConnection)c, new BufferedReader(new InputStreamReader(sql, StandardCharsets.UTF_8)), expDir);
      } catch (UnsupportedEncodingException e) {
        throw new RuntimeException(e);
      }
      // include images
      exportLogos(catalogueKey, expDir);
  
      // export citation.ini
      exportCitations(catalogueKey, expDir);
      
      // zip up archive and move to download
      File arch = new File(cfg.downloadDir, "ac-export.zip");
      logger.log("Zip up archive and move to download");
      if (arch.exists()) {
        LOG.debug("Remove previous export file {}", arch.getAbsolutePath());
      }
      LOG.info("Creating final export archive {}", arch.getAbsolutePath());
      CompressionUtil.zipDir(expDir, arch);<commit_msg>Include subdirs like logos in the export dir<commit_after>package org.col.release;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.common.io.Files;
import freemarker.template.*;
import no.api.freemarker.java8.Java8ObjectWrapper;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.col.WsServerConfig;
import org.col.api.model.Dataset;
import org.col.api.model.Page;
import org.col.api.search.DatasetSearchRequest;
import org.col.common.io.CompressionUtil;
import org.col.common.io.Utf8IOUtils;
import org.col.db.mapper.DatasetMapper;
import org.col.img.ImgConfig;
import org.col.postgres.PgCopyUtils;
import org.gbif.nameparser.api.Rank;
import org.postgresql.jdbc.PgConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class AcExporter {
  private static final Logger LOG = LoggerFactory.getLogger(AcExporter.class);
  private static final String EXPORT_SQL = "/exporter/ac-export.sql";
  private static final String COPY_WITH = "CSV HEADER NULL '\\N'";
  private static final Pattern COPY_START = Pattern.compile("^\\s*COPY\\s*\\(");
  private static final Pattern COPY_END   = Pattern.compile("^\\s*\\)\\s*TO\\s*'(.+)'");
  private static final Pattern VAR_DATASET_KEY = Pattern.compile("\\{\\{datasetKey}}", Pattern.CASE_INSENSITIVE);
  private static final Version freemarkerVersion = Configuration.VERSION_2_3_28;
  private static final Configuration fmk = new Configuration(freemarkerVersion);
  static {
    fmk.setClassForTemplateLoading(AcExporter.class, "/exporter");
    // see https://freemarker.apache.org/docs/pgui_quickstart_createconfiguration.html
    fmk.setDefaultEncoding("UTF-8");
    fmk.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
    fmk.setLogTemplateExceptions(false);
    fmk.setWrapUncheckedExceptions(true);
    // allow the use of java8 dates, see https://github.com/lazee/freemarker-java-8
    fmk.setObjectWrapper(new Java8ObjectWrapper(freemarkerVersion));
  }
  private final WsServerConfig cfg;
  private final SqlSessionFactory factory;
  private org.col.release.Logger logger;

  public AcExporter(WsServerConfig cfg, SqlSessionFactory factory) {
    this.cfg = cfg;
    this.factory = factory;
  }
  
  /**
   * @return final archive
   */
  public File export(int catalogueKey, org.col.release.Logger logger) throws IOException, SQLException, IllegalStateException {
    File expDir = new File(cfg.normalizer.scratchDir(catalogueKey), "exports");
    LOG.info("Export catalogue {} to {}", catalogueKey, expDir.getAbsolutePath());
    try {
      logger.setLog(LOG);
      this.logger = logger;
      // create csv files
      try (Connection c = cfg.db.connect()) {
        c.setAutoCommit(false);
        setupTables(c);
        InputStream sql = AcExporter.class.getResourceAsStream(EXPORT_SQL);
        executeAcExportSql(catalogueKey, (PgConnection)c, new BufferedReader(new InputStreamReader(sql, StandardCharsets.UTF_8)), expDir);
      } catch (UnsupportedEncodingException e) {
        throw new RuntimeException(e);
      }
      // include images
      exportLogos(catalogueKey, expDir);
  
      // export citation.ini
      exportCitations(catalogueKey, expDir);
      
      // zip up archive and move to download
      File arch = new File(cfg.downloadDir, "export-"+catalogueKey+".zip");
      logger.log("Zip up archive and move to download");
      if (arch.exists()) {
        LOG.debug("Remove previous export file {}", arch.getAbsolutePath());
      }
      LOG.info("Creating final export archive {}", arch.getAbsolutePath());
      CompressionUtil.zipDir(expDir, arch, true);
{'fraction_non_alphanumeric': 0.1160548757879125, 'fraction_numerical': 0.0037078235076010383, 'mean_word_length': 4.299279633267846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1368267', 'n_tokens_mistral': 2457, 'n_tokens_neox': 2459, 'n_words': 519}