<commit_before>/*
 * Copyright 2012 SFB 632.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package annis.administration;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Collectors;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;

import javax.mail.internet.AddressException;
import javax.mail.internet.InternetAddress;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;

import org.aeonbits.owner.ConfigFactory;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.FileWriterWithEncoding;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.SimpleEmail;
import org.apache.http.client.utils.URIBuilder;
import org.corpus_tools.graphannis.errors.GraphANNISException;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Multimap;
import com.google.common.io.ByteStreams;

import annis.AnnisRunnerException;
import annis.CommonHelper;
import annis.ServiceConfig;
import annis.dao.ShortenerDao;
import annis.exceptions.AnnisException;
import annis.service.objects.AnnisCorpus;
import annis.service.objects.ImportJob;
import annis.utils.ANNISFormatHelper;
import au.com.bytecode.opencsv.CSVReader;

/**
 *
 * @author <NAME> {@literal <<EMAIL>>}
 */
public class CorpusAdministration {

    private AdministrationDao administrationDao;
    private ShortenerDao shortenerDao;

    private final ServiceConfig cfg = ConfigFactory.create(ServiceConfig.class);

    private static final Logger log = LoggerFactory.getLogger(CorpusAdministration.class);

    public CorpusAdministration() {
    }

    public static CorpusAdministration create(AdministrationDao administrationDao, ShortenerDao shortenerDao) {
        CorpusAdministration corpusAdmin = new CorpusAdministration();
        corpusAdmin.setAdministrationDao(administrationDao);
        corpusAdmin.setShortenerDao(shortenerDao);

        corpusAdmin.checkDatabaseSchemaVersion();

        return corpusAdmin;
    }

    public void deleteCorpora(List<String> corpora) {
        // check if corpus exists
        List<AnnisCorpus> existingCorpora = administrationDao.getQueryDao().listCorpora(corpora);
        Set<String> existingCorpusNames = new HashSet<>();
        for (AnnisCorpus c : existingCorpora) {
            existingCorpusNames.add(c.getName());
        }
        for (String toDelete : corpora) {
            if (!existingCorpusNames.contains(toDelete)) {
                throw new AnnisRunnerException("Corpus does not exist (or is not a top-level corpus): " + toDelete, 51);
            }
        }
        log.info("Deleting corpora: " + corpora);
        getDeleteCorpusDao().deleteCorpora(corpora);
        log.info("Finished deleting corpora: " + corpora);
    }

    public void cleanupData() {
        administrationDao.cleanupData();
    }

    /**
     * Imports several corpora and catches a possible thrown
     * {@link DefaultAdministrationDao.ConflictingCorpusException} when the
     * overwrite flag is set to false.
     *
     *
     * @param overwrite
     *                              If set to false, a conflicting corpus is not
     *                              silently reimported.
     * @param aliasName
     *                              An common alias name for all imported corpora or
     *                              null
     * @param statusEmailAdress
     *                              an email adress for informating the admin about
     *                              statuses
     * @param waitForOtherTasks
     *                              If true wait for other imports to finish, if
     *                              false abort the import.
     * @param paths
     *                              Valid pathes to corpora.
     * @return True if all corpora where imported successfully.
     */
    public ImportStatus importCorporaSave(boolean overwrite, String aliasName, String statusEmailAdress,
            boolean waitForOtherTasks, List<String> paths) {

        // init the import stats. From the beginning everything is ok
        ImportStatus importStats = new ImportStatsImpl();
        importStats.setStatus(true);

        List<File> roots = new LinkedList<>();
        for (String path : paths) {
            File f = new File(path);

            if (f.isFile()) {
                // might be a ZIP-file
                try (ZipFile zip = new ZipFile(f);) {

                    // get the names of all corpora included in the ZIP file
                    // in order to get a folder name
                    Map<String, ZipEntry> corpora = ANNISFormatHelper.corporaInZipfile(zip);

                    // unzip and add all resulting corpora to import list
                    log.info("Unzipping " + f.getPath());
                    File outDir = createZIPOutputDir(Joiner.on(", ").join(corpora.keySet()));
                    roots.addAll(unzipCorpus(outDir, zip));

                } catch (ZipException ex) {
                    log.error("" + f.getAbsolutePath() + " might not be a valid ZIP file and will be ignored", ex);
                } catch (IOException ex) {
                    log.error("IOException when importing file " + f.getAbsolutePath() + ", will be ignored", ex);
                }
            } else {
                try {
                    roots.addAll(ANNISFormatHelper.corporaInDirectory(f).values());
                } catch (IOException ex) {
                    log.error("Could not find any corpus in " + f.getPath(), ex);
                    importStats.setStatus(false);
                    importStats.addException(f.getAbsolutePath(), ex);
                }
            }
        } // end for each given path

        // import each corpus separately
        for (File r : roots) {
            try {
                log.info("Importing corpus from: " + r.getPath());
                if (getAdministrationDao().importCorpus(r.getPath(), aliasName, overwrite)) {
                    log.info("Finished import from: " + r.getPath());
                    getAdministrationDao().sendImportStatusMail(statusEmailAdress, r.getPath(),
                            ImportJob.Status.SUCCESS, null);
                } else {
                    importStats.setStatus(false);
                    getAdministrationDao().sendImportStatusMail(statusEmailAdress, r.getPath(), ImportJob.Status.ERROR,
                            null);
                }
            }

            catch (AdministrationDao.ConflictingCorpusException ex) {
                importStats.setStatus(false);
                importStats.addException(r.getPath(), ex);
                log.error("Error on conflicting top level corpus name for {}", r.getPath());
                getAdministrationDao().sendImportStatusMail(statusEmailAdress, r.getPath(), ImportJob.Status.ERROR,
                        ex.getMessage());
            } catch (Throwable ex) {
                importStats.setStatus(false);
                importStats.addException(r.getPath(), ex);
                log.error("Error on importing corpus", ex);
                getAdministrationDao().sendImportStatusMail(statusEmailAdress, r.getPath(), ImportJob.Status.ERROR,
                        ex.getMessage());
            }
        } // end for each corpus

        return importStats;
    }

    public int migrateUrlShortener(List<String> paths, String serviceURL, String username, String password,
            boolean skipExisting, Multimap<QueryStatus, URLShortenerDefinition> failedQueries) {
        if (paths == null || serviceURL == null) {
            return 0;
        }

        int successfulQueries = 0;

        Client client = ClientBuilder.newClient();
        if (username != null && password != null) {
            HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basic(username, password);
            client.register(authFeature);

            // test authentication and fail early
            String result = client.target(serviceURL).path("annis").path("admin").path("is-authenticated").request()
                    .get(String.class);
            Preconditions.checkArgument("true".equalsIgnoreCase(result), "Authentication failed");
        }

        WebTarget searchService = client.target(serviceURL).path("annis").path("query").path("search");

        for (String p : paths) {
            File urlShortenerFile = new File(p);
            if (urlShortenerFile.isFile()) {
                try (CSVReader csvReader = new CSVReader(new FileReader(urlShortenerFile), '\t')) {
                    String[] line;
                    while ((line = csvReader.readNext()) != null) {
                        if (line.length == 4) {
                            
                            // parse URL
                            try {
                                URLShortenerDefinition q = URLShortenerDefinition.parse(line[3], line[0], line[2]);
                                if (q != null) {
                                    // check if all corpora exist in the new instance
                                    List<String> corpusNames = new LinkedList<>(q.getQuery().getCorpora());
                                    Set<String> knownCorpora = getAdministrationDao().getQueryDao()
                                            .listCorpora(corpusNames).stream().map((c) -> c.getName())
                                            .collect(Collectors.toSet());

                                    for (String c : corpusNames) {
                                        if (!knownCorpora.contains(c)) {
                                            q.addUnknownCorpus(c);
                                        }
                                    }

                                    if (!q.getUnknownCorpora().isEmpty()) {
                                        failedQueries.put(QueryStatus.UnknownCorpus, q);
                                    } else if (corpusNames.isEmpty()) {
                                        q.setErrorMsg("Corpus name is empty");
                                        failedQueries.put(QueryStatus.Failed, q);
                                    } else if (getShortenerDao().unshorten(q.getUuid()) != null) {
                                        if (skipExisting) {
                                            continue;
                                        } else {
                                            failedQueries.put(QueryStatus.UUIDExists, q);
                                        }
                                    } else {
                                        // check the query
                                        try {
                                            log.info("UUID {}, testing query {} on corpus {}", q.getUuid(),
                                                    q.getQuery().getQuery().trim(), q.getQuery().getCorpora());
                                            QueryStatus status = q.test(getAdministrationDao().getQueryDao(),
                                                    searchService);

                                            // insert URLs into new database
                                            String temporary = null;

                                            if (status != QueryStatus.Ok) {
                                                failedQueries.put(status, q);
                                                // Link the UUID to an error page temporarily, until the issue is
                                                // fixed.
                                                // Remember the original URL, so the temporary URL can just be set
                                                // to null to
                                                // resolve
                                                // to the original URL when the issue is fixed in ANNIS.
                                                try {
                                                    URI tempURI = new URIBuilder().setPath("/unsupported-query")
                                                            .addParameter("url", q.getUri().toASCIIString()).build();
                                                    temporary = tempURI.toASCIIString();
                                                } catch (URISyntaxException e) {
                                                    log.error("Could not create proper URI for unsupported query", e);
                                                }

                                                String lineSeparator = System.getProperty("line.separator");

                                                StringBuilder sb = new StringBuilder();
                                                sb.append("Query Error: " + status + lineSeparator);
                                                sb.append("Corpus: \"" + q.getQuery().getCorpora() + "\""
                                                        + lineSeparator);
                                                sb.append("UUID: \"" + q.getUuid() + "\"" + lineSeparator);
                                                sb.append("Query:" + lineSeparator);
                                                sb.append(q.getQuery().getQuery().trim() + lineSeparator);
                                                sb.append("Error Message: " + q.getErrorMsg());

                                                log.warn(sb.toString());
                                            }
                                            getShortenerDao().migrate(q.getUri().toASCIIString(), temporary,
                                                    "anonymous", q.getUuid(), q.getCreationTime() == null ? new Date()
                                                            : q.getCreationTime().toDate());

                                            if (status == QueryStatus.Ok) {
                                                successfulQueries++;
                                            }

                                        } catch (GraphANNISException ex) {
                                            q.setErrorMsg(ex.getMessage());
                                            failedQueries.put(QueryStatus.Failed, q);
                                        } catch (Throwable ex) {
                                            String lineSeparator = System.getProperty("line.separator");
                                            StringBuilder sb = new StringBuilder();
                                            sb.append("Query Error: " + QueryStatus.Failed + lineSeparator);
                                            sb.append("Corpus: \"" + q.getQuery().getCorpora() + "\"" + lineSeparator);
                                            sb.append("UUID: \"" + q.getUuid() + "\"" + lineSeparator);
                                            sb.append("Query:" + lineSeparator);
                                            sb.append(q.getQuery().getQuery().trim() + lineSeparator);
                                            sb.append("Error Message: " + ex.getMessage());
                                            
                                            q.setErrorMsg(ex.getMessage());

                                            log.warn(sb.toString(), ex);
                                            failedQueries.put(QueryStatus.Failed, q);
                                        }
                                    }
                                }
                            } catch (Throwable ex) {

                                String lineSeparator = System.getProperty("line.separator");

                                StringBuilder sb = new StringBuilder();
                                sb.append("Query Error: " + QueryStatus.Failed);
                                sb.append("UUID: \"" + line[0] + "\"" + lineSeparator);
                                sb.append("Error Message: " + ex.getMessage());

                                URLShortenerDefinition q = new URLShortenerDefinition(null, URLShortenerDefinition.parseUUID(line[0]),
                                        null);
                                q.setErrorMsg(ex.getMessage());
                                failedQueries.put(QueryStatus.Failed, q);

                                log.warn(sb.toString());
                            }
                        }<commit_msg>At least log the error type if error message is not available<commit_after>/*
 * Copyright 2012 SFB 632.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package annis.administration;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;

import javax.mail.internet.AddressException;
import javax.mail.internet.InternetAddress;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;

import org.aeonbits.owner.ConfigFactory;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.FileWriterWithEncoding;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.SimpleEmail;
import org.apache.http.client.utils.URIBuilder;
import org.corpus_tools.graphannis.errors.GraphANNISException;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Multimap;
import com.google.common.io.ByteStreams;

import annis.AnnisRunnerException;
import annis.CommonHelper;
import annis.ServiceConfig;
import annis.dao.ShortenerDao;
import annis.exceptions.AnnisException;
import annis.service.objects.AnnisCorpus;
import annis.service.objects.ImportJob;
import annis.utils.ANNISFormatHelper;
import au.com.bytecode.opencsv.CSVReader;

/**
 *
 * @author <NAME> {@literal <<EMAIL>>}
 */
public class CorpusAdministration {

    private AdministrationDao administrationDao;
    private ShortenerDao shortenerDao;

    private final ServiceConfig cfg = ConfigFactory.create(ServiceConfig.class);

    private static final Logger log = LoggerFactory.getLogger(CorpusAdministration.class);

    public CorpusAdministration() {
    }

    public static CorpusAdministration create(AdministrationDao administrationDao, ShortenerDao shortenerDao) {
        CorpusAdministration corpusAdmin = new CorpusAdministration();
        corpusAdmin.setAdministrationDao(administrationDao);
        corpusAdmin.setShortenerDao(shortenerDao);

        corpusAdmin.checkDatabaseSchemaVersion();

        return corpusAdmin;
    }

    public void deleteCorpora(List<String> corpora) {
        // check if corpus exists
        List<AnnisCorpus> existingCorpora = administrationDao.getQueryDao().listCorpora(corpora);
        Set<String> existingCorpusNames = new HashSet<>();
        for (AnnisCorpus c : existingCorpora) {
            existingCorpusNames.add(c.getName());
        }
        for (String toDelete : corpora) {
            if (!existingCorpusNames.contains(toDelete)) {
                throw new AnnisRunnerException("Corpus does not exist (or is not a top-level corpus): " + toDelete, 51);
            }
        }
        log.info("Deleting corpora: " + corpora);
        getDeleteCorpusDao().deleteCorpora(corpora);
        log.info("Finished deleting corpora: " + corpora);
    }

    public void cleanupData() {
        administrationDao.cleanupData();
    }

    /**
     * Imports several corpora and catches a possible thrown
     * {@link DefaultAdministrationDao.ConflictingCorpusException} when the
     * overwrite flag is set to false.
     *
     *
     * @param overwrite
     *                              If set to false, a conflicting corpus is not
     *                              silently reimported.
     * @param aliasName
     *                              An common alias name for all imported corpora or
     *                              null
     * @param statusEmailAdress
     *                              an email adress for informating the admin about
     *                              statuses
     * @param waitForOtherTasks
     *                              If true wait for other imports to finish, if
     *                              false abort the import.
     * @param paths
     *                              Valid pathes to corpora.
     * @return True if all corpora where imported successfully.
     */
    public ImportStatus importCorporaSave(boolean overwrite, String aliasName, String statusEmailAdress,
            boolean waitForOtherTasks, List<String> paths) {

        // init the import stats. From the beginning everything is ok
        ImportStatus importStats = new ImportStatsImpl();
        importStats.setStatus(true);

        List<File> roots = new LinkedList<>();
        for (String path : paths) {
            File f = new File(path);

            if (f.isFile()) {
                // might be a ZIP-file
                try (ZipFile zip = new ZipFile(f);) {

                    // get the names of all corpora included in the ZIP file
                    // in order to get a folder name
                    Map<String, ZipEntry> corpora = ANNISFormatHelper.corporaInZipfile(zip);

                    // unzip and add all resulting corpora to import list
                    log.info("Unzipping " + f.getPath());
                    File outDir = createZIPOutputDir(Joiner.on(", ").join(corpora.keySet()));
                    roots.addAll(unzipCorpus(outDir, zip));

                } catch (ZipException ex) {
                    log.error("" + f.getAbsolutePath() + " might not be a valid ZIP file and will be ignored", ex);
                } catch (IOException ex) {
                    log.error("IOException when importing file " + f.getAbsolutePath() + ", will be ignored", ex);
                }
            } else {
                try {
                    roots.addAll(ANNISFormatHelper.corporaInDirectory(f).values());
                } catch (IOException ex) {
                    log.error("Could not find any corpus in " + f.getPath(), ex);
                    importStats.setStatus(false);
                    importStats.addException(f.getAbsolutePath(), ex);
                }
            }
        } // end for each given path

        // import each corpus separately
        for (File r : roots) {
            try {
                log.info("Importing corpus from: " + r.getPath());
                if (getAdministrationDao().importCorpus(r.getPath(), aliasName, overwrite)) {
                    log.info("Finished import from: " + r.getPath());
                    getAdministrationDao().sendImportStatusMail(statusEmailAdress, r.getPath(),
                            ImportJob.Status.SUCCESS, null);
                } else {
                    importStats.setStatus(false);
                    getAdministrationDao().sendImportStatusMail(statusEmailAdress, r.getPath(), ImportJob.Status.ERROR,
                            null);
                }
            }

            catch (AdministrationDao.ConflictingCorpusException ex) {
                importStats.setStatus(false);
                importStats.addException(r.getPath(), ex);
                log.error("Error on conflicting top level corpus name for {}", r.getPath());
                getAdministrationDao().sendImportStatusMail(statusEmailAdress, r.getPath(), ImportJob.Status.ERROR,
                        ex.getMessage());
            } catch (Throwable ex) {
                importStats.setStatus(false);
                importStats.addException(r.getPath(), ex);
                log.error("Error on importing corpus", ex);
                getAdministrationDao().sendImportStatusMail(statusEmailAdress, r.getPath(), ImportJob.Status.ERROR,
                        ex.getMessage());
            }
        } // end for each corpus

        return importStats;
    }

    public int migrateUrlShortener(List<String> paths, String serviceURL, String username, String password,
            boolean skipExisting, Multimap<QueryStatus, URLShortenerDefinition> failedQueries) {
        if (paths == null || serviceURL == null) {
            return 0;
        }

        int successfulQueries = 0;

        Client client = ClientBuilder.newClient();
        if (username != null && password != null) {
            HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basic(username, password);
            client.register(authFeature);

            // test authentication and fail early
            String result = client.target(serviceURL).path("annis").path("admin").path("is-authenticated").request()
                    .get(String.class);
            Preconditions.checkArgument("true".equalsIgnoreCase(result), "Authentication failed");
        }

        WebTarget searchService = client.target(serviceURL).path("annis").path("query").path("search");

        for (String p : paths) {
            File urlShortenerFile = new File(p);
            if (urlShortenerFile.isFile()) {
                try (CSVReader csvReader = new CSVReader(new FileReader(urlShortenerFile), '\t')) {
                    String[] line;
                    while ((line = csvReader.readNext()) != null) {
                        if (line.length == 4) {
                            
                            // parse URL
                            try {
                                URLShortenerDefinition q = URLShortenerDefinition.parse(line[3], line[0], line[2]);
                                if (q != null) {
                                    // check if all corpora exist in the new instance
                                    List<String> corpusNames = new LinkedList<>(q.getQuery().getCorpora());
                                    Set<String> knownCorpora = getAdministrationDao().getQueryDao()
                                            .listCorpora(corpusNames).stream().map((c) -> c.getName())
                                            .collect(Collectors.toSet());

                                    for (String c : corpusNames) {
                                        if (!knownCorpora.contains(c)) {
                                            q.addUnknownCorpus(c);
                                        }
                                    }

                                    if (!q.getUnknownCorpora().isEmpty()) {
                                        failedQueries.put(QueryStatus.UnknownCorpus, q);
                                    } else if (corpusNames.isEmpty()) {
                                        q.setErrorMsg("Corpus name is empty");
                                        failedQueries.put(QueryStatus.Failed, q);
                                    } else if (getShortenerDao().unshorten(q.getUuid()) != null) {
                                        if (skipExisting) {
                                            continue;
                                        } else {
                                            failedQueries.put(QueryStatus.UUIDExists, q);
                                        }
                                    } else {
                                        // check the query
                                        try {
                                            log.info("UUID {}, testing query {} on corpus {}", q.getUuid(),
                                                    q.getQuery().getQuery().trim(), q.getQuery().getCorpora());
                                            QueryStatus status = q.test(getAdministrationDao().getQueryDao(),
                                                    searchService);

                                            // insert URLs into new database
                                            String temporary = null;

                                            if (status != QueryStatus.Ok) {
                                                failedQueries.put(status, q);
                                                // Link the UUID to an error page temporarily, until the issue is
                                                // fixed.
                                                // Remember the original URL, so the temporary URL can just be set
                                                // to null to
                                                // resolve
                                                // to the original URL when the issue is fixed in ANNIS.
                                                try {
                                                    URI tempURI = new URIBuilder().setPath("/unsupported-query")
                                                            .addParameter("url", q.getUri().toASCIIString()).build();
                                                    temporary = tempURI.toASCIIString();
                                                } catch (URISyntaxException e) {
                                                    log.error("Could not create proper URI for unsupported query", e);
                                                }

                                                String lineSeparator = System.getProperty("line.separator");

                                                StringBuilder sb = new StringBuilder();
                                                sb.append("Query Error: " + status + lineSeparator);
                                                sb.append("Corpus: \"" + q.getQuery().getCorpora() + "\""
                                                        + lineSeparator);
                                                sb.append("UUID: \"" + q.getUuid() + "\"" + lineSeparator);
                                                sb.append("Query:" + lineSeparator);
                                                sb.append(q.getQuery().getQuery().trim() + lineSeparator);
                                                sb.append("Error Message: " + q.getErrorMsg());

                                                log.warn(sb.toString());
                                            }
                                            getShortenerDao().migrate(q.getUri().toASCIIString(), temporary,
                                                    "anonymous", q.getUuid(), q.getCreationTime() == null ? new Date()
                                                            : q.getCreationTime().toDate());

                                            if (status == QueryStatus.Ok) {
                                                successfulQueries++;
                                            }

                                        } catch (GraphANNISException ex) {
                                            q.setErrorMsg(ex.getMessage());
                                            failedQueries.put(QueryStatus.Failed, q);
                                        } catch (Throwable ex) {
                                            String lineSeparator = System.getProperty("line.separator");
                                            StringBuilder sb = new StringBuilder();
                                            sb.append("Query Error: " + QueryStatus.Failed + lineSeparator);
                                            sb.append("Corpus: \"" + q.getQuery().getCorpora() + "\"" + lineSeparator);
                                            sb.append("UUID: \"" + q.getUuid() + "\"" + lineSeparator);
                                            sb.append("Query:" + lineSeparator);
                                            sb.append(q.getQuery().getQuery().trim() + lineSeparator);
                                            sb.append("Error Message: " + ex.getMessage());
                                            
                                            q.setErrorMsg(ex.getMessage());

                                            log.warn(sb.toString(), ex);
                                            failedQueries.put(QueryStatus.Failed, q);
                                        }
                                    }
                                }
                            } catch (Throwable ex) {

                                String lineSeparator = System.getProperty("line.separator");
                                
                                String errorMsg = ex.getMessage();
                                if(errorMsg == null) {
                                    errorMsg = ex.getClass().toString();
                                }

                                StringBuilder sb = new StringBuilder();
                                sb.append("Query Error: " + QueryStatus.Failed + lineSeparator);
                                sb.append("UUID: \"" + line[0] + "\"" + lineSeparator);
                                sb.append("Error Message: " + errorMsg);

                                URLShortenerDefinition q = new URLShortenerDefinition(null, URLShortenerDefinition.parseUUID(line[0]),
                                        null);
                                q.setErrorMsg(errorMsg);
                                failedQueries.put(QueryStatus.Failed, q);

                                log.warn(sb.toString());
                            }
                        }
{'fraction_non_alphanumeric': 0.08968320717689936, 'fraction_numerical': 0.0012895991028875806, 'mean_word_length': 1.2951357611632994, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 313, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3635523', 'n_tokens_mistral': 7834, 'n_tokens_neox': 7557, 'n_words': 1841}