<commit_before>/*
 * Copyright 2012 SFB 632.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package annis.administration;

import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.springframework.transaction.annotation.Transactional;
import annis.AnnisRunnerException;
import annis.administration.AdministrationDao.ImportStats;
import annis.exceptions.AnnisException;
import annis.service.objects.ImportJob;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import javax.mail.internet.InternetAddress;
import org.apache.commons.io.output.FileWriterWithEncoding;
import org.apache.commons.mail.SimpleEmail;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class CorpusAdministration
{

  private AdministrationDao administrationDao;
  private SchemeFixer schemeFixer;
  private String statusMailSender;

  private static final Logger log = LoggerFactory.getLogger(
    CorpusAdministration.class);

  public CorpusAdministration()
  {
  }

  @Transactional(readOnly = false)
  public void deleteCorpora(List<Long> ids)
  {
    // check if corpus exists
    List<Long> corpora = administrationDao.listToplevelCorpora();
    for (long id : ids)
    {
      if (!corpora.contains(id))
      {
        throw new AnnisRunnerException(
          "Corpus does not exist (or is not a top-level corpus): "
          + id);
      }
    }
    log.info("Deleting corpora: " + ids);
    administrationDao.deleteCorpora(ids, true);
    log.info("Finished deleting corpora: " + ids);
  }

  public void initializeDatabase(String host, String port, String database,
    String user, String password, String defaultDatabase, String superUser,
    String superPassword, boolean useSSL)
  {

    log.info("initializing database");
    administrationDao.initializeDatabase(host, port, database, user, password,
      defaultDatabase, superUser, superPassword, useSSL);

    // write database information to property file
    writeDatabasePropertiesFile(host, port, database, user, password, useSSL);
    
    // create tables and other stuff that is handled by the scheme fixer
    if(schemeFixer != null)
    {
      schemeFixer.checkAndFix();
    }
  }

  /**
   * Imports several corpora and catches a possible thrown
   * {@link DefaultAdministrationDao.ConflictingCorpusException} when the
   * overwrite flag is set to false.
   *
   *
   * @param overwrite If set to false, a conflicting corpus is not silently
   * reimported.
   * @param aliasName An common alias name for all imported corpora or null
   * @param statusEmailAdress an email adress for informating the admin about
   * statuses
   * @param waitForOtherTasks If true wait for other imports to finish, if false
   * abort the import.
   * @param paths Valid pathes to corpora.
   * @return True if all corpora where imported successfully.
   */  
  public ImportStats importCorporaSave(boolean overwrite,
    String aliasName,
    String statusEmailAdress, boolean waitForOtherTasks, List<String> paths)
  {

    // init the import stats. From the beginning everything is ok
    ImportStats importStats = new ImportStatsImpl();
    importStats.setStatus(true);

    // check if database scheme is ok
    if(schemeFixer != null)
    {
      schemeFixer.checkAndFix();
    }
    
    // import each corpus
    for (String path : paths)
    {
      try
      {
        log.info("Importing corpus from: " + path);
        if(administrationDao.importCorpus(path, aliasName,
          overwrite, waitForOtherTasks))
        {
          log.info("Finished import from: " + path);
          sendStatusMail(statusEmailAdress, path, ImportJob.Status.SUCCESS, null);
        }
        else
        {
          importStats.setStatus(false);
          sendStatusMail(statusEmailAdress, path, ImportJob.Status.ERROR, null);
        }
      }
      catch (DefaultAdministrationDao.ConflictingCorpusException ex)
      {
        importStats.setStatus(false);
        importStats.addException(path, ex);
        log.error("Error on conflicting top level corpus name");
        sendStatusMail(statusEmailAdress, path, ImportJob.Status.ERROR, ex.
          getMessage());
      }
      catch (Throwable ex)
      {
        importStats.setStatus(false);
        importStats.addException(path, ex);
        log.error("Error on importing corpus", ex);
        sendStatusMail(statusEmailAdress, path, ImportJob.Status.ERROR, ex.
          getMessage());
      }
    }

    return importStats;
  }

  private class ImportStatsImpl implements AdministrationDao.ImportStats
  {

    private boolean status;

    private final Map<String, List<Throwable>> exceptions;<commit_msg>More user friendly message when postgres is not available when importing corpora.<commit_after>/*
 * Copyright 2012 SFB 632.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package annis.administration;

import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.springframework.transaction.annotation.Transactional;
import annis.AnnisRunnerException;
import annis.administration.AdministrationDao.ImportStats;
import annis.exceptions.AnnisException;
import annis.service.objects.ImportJob;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import javax.mail.internet.InternetAddress;
import org.apache.commons.io.output.FileWriterWithEncoding;
import org.apache.commons.mail.SimpleEmail;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class CorpusAdministration
{

  private AdministrationDao administrationDao;
  private SchemeFixer schemeFixer;
  private String statusMailSender;

  private static final Logger log = LoggerFactory.getLogger(
    CorpusAdministration.class);

  public CorpusAdministration()
  {
  }

  @Transactional(readOnly = false)
  public void deleteCorpora(List<Long> ids)
  {
    // check if corpus exists
    List<Long> corpora = administrationDao.listToplevelCorpora();
    for (long id : ids)
    {
      if (!corpora.contains(id))
      {
        throw new AnnisRunnerException(
          "Corpus does not exist (or is not a top-level corpus): "
          + id);
      }
    }
    log.info("Deleting corpora: " + ids);
    administrationDao.deleteCorpora(ids, true);
    log.info("Finished deleting corpora: " + ids);
  }

  public void initializeDatabase(String host, String port, String database,
    String user, String password, String defaultDatabase, String superUser,
    String superPassword, boolean useSSL)
  {

    log.info("initializing database");
    administrationDao.initializeDatabase(host, port, database, user, password,
      defaultDatabase, superUser, superPassword, useSSL);

    // write database information to property file
    writeDatabasePropertiesFile(host, port, database, user, password, useSSL);
    
    // create tables and other stuff that is handled by the scheme fixer
    if(schemeFixer != null)
    {
      schemeFixer.checkAndFix();
    }
  }

  /**
   * Imports several corpora and catches a possible thrown
   * {@link DefaultAdministrationDao.ConflictingCorpusException} when the
   * overwrite flag is set to false.
   *
   *
   * @param overwrite If set to false, a conflicting corpus is not silently
   * reimported.
   * @param aliasName An common alias name for all imported corpora or null
   * @param statusEmailAdress an email adress for informating the admin about
   * statuses
   * @param waitForOtherTasks If true wait for other imports to finish, if false
   * abort the import.
   * @param paths Valid pathes to corpora.
   * @return True if all corpora where imported successfully.
   */  
  public ImportStats importCorporaSave(boolean overwrite,
    String aliasName,
    String statusEmailAdress, boolean waitForOtherTasks, List<String> paths)
  {

    // init the import stats. From the beginning everything is ok
    ImportStats importStats = new ImportStatsImpl();
    importStats.setStatus(true);

    // check if database scheme is ok
    if(schemeFixer != null)
    {
      schemeFixer.checkAndFix();
    }
    
    // import each corpus
    for (String path : paths)
    {
      try
      {
        log.info("Importing corpus from: " + path);
        if(administrationDao.importCorpus(path, aliasName,
          overwrite, waitForOtherTasks))
        {
          log.info("Finished import from: " + path);
          sendStatusMail(statusEmailAdress, path, ImportJob.Status.SUCCESS, null);
        }
        else
        {
          importStats.setStatus(false);
          sendStatusMail(statusEmailAdress, path, ImportJob.Status.ERROR, null);
        }
      }
      catch (DefaultAdministrationDao.ConflictingCorpusException ex)
      {
        importStats.setStatus(false);
        importStats.addException(path, ex);
        log.error("Error on conflicting top level corpus name");
        sendStatusMail(statusEmailAdress, path, ImportJob.Status.ERROR, ex.
          getMessage());
      }
      catch (org.springframework.transaction.CannotCreateTransactionException ex)
      {
        log.error("Postgres is not running or misconfigured");
      }
      catch (Throwable ex)
      {
        importStats.setStatus(false);
        importStats.addException(path, ex);
        log.error("Error on importing corpus", ex);
        sendStatusMail(statusEmailAdress, path, ImportJob.Status.ERROR, ex.
          getMessage());
      }
    }

    return importStats;
  }

  private class ImportStatsImpl implements AdministrationDao.ImportStats
  {

    private boolean status;

    private final Map<String, List<Throwable>> exceptions;
{'fraction_non_alphanumeric': 0.08495346908688842, 'fraction_numerical': 0.0023956509720814523, 'mean_word_length': 3.2969121140142517, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5219431', 'n_tokens_mistral': 3023, 'n_tokens_neox': 2940, 'n_words': 909}