<commit_before>/*
 * Copyright 2009-2011 Collaborative Research Centre SFB 632
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.hu_berlin.german.korpling.annis.kickstarter;

import annis.administration.AdministrationDao;
import annis.administration.AdministrationDao.StatementController;
import annis.administration.CorpusAdministration;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.AppenderBase;
import java.io.*;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author thomas
 */
public class ImportDialog extends javax.swing.JDialog
{

  private static final org.slf4j.Logger log =
    LoggerFactory.getLogger(ImportDialog.class);

  private File confFile;

  private Properties confProps;

  private StatementController statementController = new StatementControllerImpl();

  private static class Status
  {

    public boolean ok = true;

    public Exception ex = new Exception();

  }

  private class StatementControllerImpl implements StatementController
  {

    PreparedStatement statement = null;

    Boolean isCancelled = false;

    @Override
    public void registerStatement(PreparedStatement statement)
    {

      this.statement = statement;
      if (isCancelled)
      {
        cancelStatements();
      }

    }

    @Override
    public void cancelStatements()
    {
      isCancelled = true;
      if (statement != null)
      {
        try
        {
          log.info("cancel statement");
          statement.cancel();
        }
        catch (SQLException ex)
        {
          log.error("problems with interrupting statement", ex);
        }
      }

    }

    @Override
    public boolean isCancelled()
    {
      return isCancelled;
    }
  }

  private class ImportDialogWorker extends SwingWorker<Status, Void> implements
    Serializable
  {

    StatementController statementController;

    public ImportDialogWorker(StatementController statementController)
    {
      this.statementController = statementController;
    }

    public void cancelStatement()
    {
      statementController.cancelStatements();
    }

    @Override
    protected Status doInBackground() throws Exception
    {
      Status status = new Status();
      AdministrationDao.ImportStats importStats = null;
      StringBuilder errorMessages = new StringBuilder();

      if (corpora == null)
      {
        try
        {
          SwingUtilities.invokeLater(new Runnable()
          {
            @Override
            public void run()
            {
              lblCurrentCorpus.setText("import " + StringUtils.abbreviateMiddle(
                txtInputDir.getText(), "...", 50));
              pbCorpus.setMaximum(1);
              pbCorpus.setMinimum(0);
              pbCorpus.setValue(0);
            }
          });

          corpusAdministration.getAdministrationDao().registerGUICancelThread(
            statementController);

          importStats = corpusAdministration.importCorporaSave(
            jCheckBox1.isSelected(), null, null, false, txtInputDir.getText());

        }
        catch (Exception ex)
        {
          status.ok = false;
          status.ex = ex;
          return status;
        }
      }
      else
      {
        SwingUtilities.invokeLater(new Runnable()
        {
          @Override
          public void run()
          {
            pbCorpus.setMaximum(corpora.size());
            pbCorpus.setMinimum(0);
            pbCorpus.setValue(0);
          }
        });

        int i = 0;
        for (Map<String, Object> corpus : corpora)
        {
          if (isCancelled())
          {
            status.ok = true;
            return status;
          }

          if (corpus.containsKey("source_path"))
          {
            final String path = (String) corpus.get("source_path");

            final int finalI = i;
            SwingUtilities.invokeLater(new Runnable()
            {
              @Override
              public void run()
              {
                lblCurrentCorpus.setText("import "
                  + StringUtils.abbreviateMiddle(path, "...", 40)
                  + " [" + (finalI + 1) + "/" + corpora.size() + "]");
                pbCorpus.setValue(finalI);
              }
            });


            try
            {

              corpusAdministration.getAdministrationDao().
                registerGUICancelThread(statementController);

              if (importStats == null)
              {
              importStats = corpusAdministration.importCorporaSave(
                jCheckBox1.isSelected(), null, null, false, path);
              } else {
                importStats.add(corpusAdministration.importCorporaSave(
                  jCheckBox1.isSelected(), null, null, false, path));
              }
            }
            catch (Exception ex)
            {
              log.error("could not import corpus", ex);
              errorMessages.append("[").append(path).append("]\n");
              errorMessages.append(ex.getMessage()).append(path).append("\n\n");
            }
          }
          i++;
        }
      }

      if (!importStats.getStatus())
      {
        status.ok = false;
        status.ex = (Exception) importStats.getExceptions().get(importStats.getExceptions().size() -1);
        return status;
      }

      return status;
    }

    @Override
    protected void done()
    {
      isImporting = false;
      btOk.setEnabled(true);
      btSearchInputDir.setEnabled(true);
      txtInputDir.setEnabled(true);
      lblCurrentCorpus.setText("");
      pbCorpus.setValue(pbCorpus.getMaximum());
      pbImport.setIndeterminate(false);

      try
      {
        if (!isCancelled())
        {
          Status status = this.get();
          if (status.ok)
          {
            JOptionPane.showMessageDialog(null,
              "Corpus imported.", "INFO", JOptionPane.INFORMATION_MESSAGE);
            setVisible(false);
          }
          else
          {
            new ExceptionDialog(status.ex, "Import failed").setVisible(true);
            setVisible(false);
          }
        }
      }
      catch (Exception ex)
      {
        log.error(null, ex);
      }
    }
  }
  private CorpusAdministration corpusAdministration;

  private SwingWorker<Status, Void> worker;

  private boolean isImporting;

  private List<Map<String, Object>> corpora;

  public ImportDialog(java.awt.Frame parent, boolean modal,
    CorpusAdministration corpusAdmin)
  {
    this(parent, modal, corpusAdmin, null);
  }

  /**
   * Creates new form ImportDialog
   */
  public ImportDialog(java.awt.Frame parent, boolean modal,
    CorpusAdministration corpusAdmin, List<Map<String, Object>> corpora)
  {
    super(parent, modal);

    this.corpusAdministration = corpusAdmin;
    this.corpora = corpora;

    confProps = new Properties();
    confFile = new File(System.getProperty("user.home")
      + "/.annis/kickstart.properties");
    try
    {
      if (!confFile.exists())
      {
        if (!confFile.getParentFile().mkdirs())
        {
          log.warn("Cannot create directory " + confFile.getAbsolutePath());
        }
        if (!confFile.createNewFile())
        {
          log.warn("Cannot create file " + confFile.getAbsolutePath());
        }
      }
    }
    catch (IOException ex)
    {
      log.error(null, ex);
    }

    initComponents();

    loadProperties();

    getRootPane().setDefaultButton(btOk);

    isImporting = false;
    worker = new ImportDialogWorker(statementController);

    addAppender();

    // directly start import if we were called from outside
    if (this.corpora != null)
    {
      startImport();
    }

  }

  private void storeProperties()
  {
    confProps.put("last-directory", txtInputDir.getText());
    FileOutputStream oStream = null;
    try
    {
      oStream = new FileOutputStream(confFile);
      confProps.store(oStream, "");
    }
    catch (IOException ex)
    {
      log.error(null, ex);
    }
    finally
    {
      if (oStream != null)
      {
        try
        {
          oStream.close();
        }
        catch (IOException ex)
        {
          log.error(null, ex);
        }
      }
    }
  }

  private void loadProperties()
  {
    FileInputStream iStream = null;
    try
    {
      iStream = new FileInputStream(confFile);
      confProps.load(iStream);
      String lastDirectory = confProps.getProperty("last-directory");
      if (lastDirectory != null)
      {
        txtInputDir.setText(lastDirectory);
      }

    }
    catch (IOException ex)
    {
      log.error(null, ex);
    }
    finally
    {
      if (iStream != null)
      {
        try
        {
          iStream.close();
        }
        catch (IOException ex)
        {
          log.error(null, ex);
        }
      }
    }

  }

  private void addAppender()
  {
    LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
    JoranConfigurator jc = new JoranConfigurator();
    jc.setContext(lc);


    Appender appender = new AppenderBase<ILoggingEvent>()
    {
      @Override
      protected void append(ILoggingEvent event)
      {
        if (event.getLevel().isGreaterOrEqual(Level.INFO))
        {
          lblStatus.setText(event.getMessage());<commit_msg>also output formatted messages in Kickstarter<commit_after>/*
 * Copyright 2009-2011 Collaborative Research Centre SFB 632
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.hu_berlin.german.korpling.annis.kickstarter;

import annis.administration.AdministrationDao;
import annis.administration.AdministrationDao.StatementController;
import annis.administration.CorpusAdministration;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.AppenderBase;
import java.io.*;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author thomas
 */
public class ImportDialog extends javax.swing.JDialog
{

  private static final org.slf4j.Logger log =
    LoggerFactory.getLogger(ImportDialog.class);

  private File confFile;

  private Properties confProps;

  private StatementController statementController = new StatementControllerImpl();

  private static class Status
  {

    public boolean ok = true;

    public Exception ex = new Exception();

  }

  private class StatementControllerImpl implements StatementController
  {

    PreparedStatement statement = null;

    Boolean isCancelled = false;

    @Override
    public void registerStatement(PreparedStatement statement)
    {

      this.statement = statement;
      if (isCancelled)
      {
        cancelStatements();
      }

    }

    @Override
    public void cancelStatements()
    {
      isCancelled = true;
      if (statement != null)
      {
        try
        {
          log.info("cancel statement");
          statement.cancel();
        }
        catch (SQLException ex)
        {
          log.error("problems with interrupting statement", ex);
        }
      }

    }

    @Override
    public boolean isCancelled()
    {
      return isCancelled;
    }
  }

  private class ImportDialogWorker extends SwingWorker<Status, Void> implements
    Serializable
  {

    StatementController statementController;

    public ImportDialogWorker(StatementController statementController)
    {
      this.statementController = statementController;
    }

    public void cancelStatement()
    {
      statementController.cancelStatements();
    }

    @Override
    protected Status doInBackground() throws Exception
    {
      Status status = new Status();
      AdministrationDao.ImportStats importStats = null;
      StringBuilder errorMessages = new StringBuilder();

      if (corpora == null)
      {
        try
        {
          SwingUtilities.invokeLater(new Runnable()
          {
            @Override
            public void run()
            {
              lblCurrentCorpus.setText("import " + StringUtils.abbreviateMiddle(
                txtInputDir.getText(), "...", 50));
              pbCorpus.setMaximum(1);
              pbCorpus.setMinimum(0);
              pbCorpus.setValue(0);
            }
          });

          corpusAdministration.getAdministrationDao().registerGUICancelThread(
            statementController);

          importStats = corpusAdministration.importCorporaSave(
            jCheckBox1.isSelected(), null, null, false, txtInputDir.getText());

        }
        catch (Exception ex)
        {
          status.ok = false;
          status.ex = ex;
          return status;
        }
      }
      else
      {
        SwingUtilities.invokeLater(new Runnable()
        {
          @Override
          public void run()
          {
            pbCorpus.setMaximum(corpora.size());
            pbCorpus.setMinimum(0);
            pbCorpus.setValue(0);
          }
        });

        int i = 0;
        for (Map<String, Object> corpus : corpora)
        {
          if (isCancelled())
          {
            status.ok = true;
            return status;
          }

          if (corpus.containsKey("source_path"))
          {
            final String path = (String) corpus.get("source_path");

            final int finalI = i;
            SwingUtilities.invokeLater(new Runnable()
            {
              @Override
              public void run()
              {
                lblCurrentCorpus.setText("import "
                  + StringUtils.abbreviateMiddle(path, "...", 40)
                  + " [" + (finalI + 1) + "/" + corpora.size() + "]");
                pbCorpus.setValue(finalI);
              }
            });


            try
            {

              corpusAdministration.getAdministrationDao().
                registerGUICancelThread(statementController);

              if (importStats == null)
              {
              importStats = corpusAdministration.importCorporaSave(
                jCheckBox1.isSelected(), null, null, false, path);
              } else {
                importStats.add(corpusAdministration.importCorporaSave(
                  jCheckBox1.isSelected(), null, null, false, path));
              }
            }
            catch (Exception ex)
            {
              log.error("could not import corpus", ex);
              errorMessages.append("[").append(path).append("]\n");
              errorMessages.append(ex.getMessage()).append(path).append("\n\n");
            }
          }
          i++;
        }
      }

      if (!importStats.getStatus())
      {
        status.ok = false;
        status.ex = (Exception) importStats.getExceptions().get(importStats.getExceptions().size() -1);
        return status;
      }

      return status;
    }

    @Override
    protected void done()
    {
      isImporting = false;
      btOk.setEnabled(true);
      btSearchInputDir.setEnabled(true);
      txtInputDir.setEnabled(true);
      lblCurrentCorpus.setText("");
      pbCorpus.setValue(pbCorpus.getMaximum());
      pbImport.setIndeterminate(false);

      try
      {
        if (!isCancelled())
        {
          Status status = this.get();
          if (status.ok)
          {
            JOptionPane.showMessageDialog(null,
              "Corpus imported.", "INFO", JOptionPane.INFORMATION_MESSAGE);
            setVisible(false);
          }
          else
          {
            new ExceptionDialog(status.ex, "Import failed").setVisible(true);
            setVisible(false);
          }
        }
      }
      catch (Exception ex)
      {
        log.error(null, ex);
      }
    }
  }
  private CorpusAdministration corpusAdministration;

  private SwingWorker<Status, Void> worker;

  private boolean isImporting;

  private List<Map<String, Object>> corpora;

  public ImportDialog(java.awt.Frame parent, boolean modal,
    CorpusAdministration corpusAdmin)
  {
    this(parent, modal, corpusAdmin, null);
  }

  /**
   * Creates new form ImportDialog
   */
  public ImportDialog(java.awt.Frame parent, boolean modal,
    CorpusAdministration corpusAdmin, List<Map<String, Object>> corpora)
  {
    super(parent, modal);

    this.corpusAdministration = corpusAdmin;
    this.corpora = corpora;

    confProps = new Properties();
    confFile = new File(System.getProperty("user.home")
      + "/.annis/kickstart.properties");
    try
    {
      if (!confFile.exists())
      {
        if (!confFile.getParentFile().mkdirs())
        {
          log.warn("Cannot create directory " + confFile.getAbsolutePath());
        }
        if (!confFile.createNewFile())
        {
          log.warn("Cannot create file " + confFile.getAbsolutePath());
        }
      }
    }
    catch (IOException ex)
    {
      log.error(null, ex);
    }

    initComponents();

    loadProperties();

    getRootPane().setDefaultButton(btOk);

    isImporting = false;
    worker = new ImportDialogWorker(statementController);

    addAppender();

    // directly start import if we were called from outside
    if (this.corpora != null)
    {
      startImport();
    }

  }

  private void storeProperties()
  {
    confProps.put("last-directory", txtInputDir.getText());
    FileOutputStream oStream = null;
    try
    {
      oStream = new FileOutputStream(confFile);
      confProps.store(oStream, "");
    }
    catch (IOException ex)
    {
      log.error(null, ex);
    }
    finally
    {
      if (oStream != null)
      {
        try
        {
          oStream.close();
        }
        catch (IOException ex)
        {
          log.error(null, ex);
        }
      }
    }
  }

  private void loadProperties()
  {
    FileInputStream iStream = null;
    try
    {
      iStream = new FileInputStream(confFile);
      confProps.load(iStream);
      String lastDirectory = confProps.getProperty("last-directory");
      if (lastDirectory != null)
      {
        txtInputDir.setText(lastDirectory);
      }

    }
    catch (IOException ex)
    {
      log.error(null, ex);
    }
    finally
    {
      if (iStream != null)
      {
        try
        {
          iStream.close();
        }
        catch (IOException ex)
        {
          log.error(null, ex);
        }
      }
    }

  }

  private void addAppender()
  {
    LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
    JoranConfigurator jc = new JoranConfigurator();
    jc.setContext(lc);


    Appender appender = new AppenderBase<ILoggingEvent>()
    {
      @Override
      protected void append(ILoggingEvent event)
      {
        if (event.getLevel().isGreaterOrEqual(Level.INFO))
        {
          lblStatus.setText(event.getFormattedMessage());
{'fraction_non_alphanumeric': 0.10089827628065064, 'fraction_numerical': 0.0033017722748239866, 'mean_word_length': 2.0749477455957, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6821495', 'n_tokens_mistral': 5863, 'n_tokens_neox': 5616, 'n_words': 1202}