<commit_before>/* This file is part of VoltDB.
 * Copyright (C) 2008-2013 VoltDB Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.voltdb.utils;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

import org.hsqldb_voltpatches.lib.tar.TarGenerator;
import org.hsqldb_voltpatches.lib.tar.TarMalformatException;
import org.json_voltpatches.JSONArray;
import org.json_voltpatches.JSONException;
import org.json_voltpatches.JSONObject;
import org.json_voltpatches.JSONStringer;
import org.voltdb.processtools.SFTPSession;
import org.voltdb.processtools.SFTPSession.SFTPException;
import org.voltdb.processtools.SSHTools;
import org.voltdb.types.TimestampType;

import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.net.HostAndPort;

public class Collector {
    private static String m_voltDbRootPath = null;
    private static String m_configInfoPath = null;
    private static String m_catalogJarPath = null;
    private static String m_deploymentPath = null;

    private static String m_uniqueid = "";
    private static String m_host = "";
    private static String m_username = "";
    private static String m_password = "";
    private static boolean m_noPrompt = false;
    private static boolean m_dryRun = false;
    private static boolean m_noHeapDump = false;
    private static boolean m_calledFromVEM = false;
    private static boolean m_fileInfoOnly = false;

    private static int m_pid = 0;
    private static String m_workingDir = null;
    private static List<String> m_logPaths = new ArrayList<String>();

    public static String[] cmdFilenames = {"sardata", "dmesgdata"};

    public static void main(String[] args) {
        m_voltDbRootPath = args[0];
        m_uniqueid = args[1];
        m_host = args[2];
        m_username = args[3];
        m_password = args[4];
        m_noPrompt = Boolean.parseBoolean(args[5]);
        m_dryRun = Boolean.parseBoolean(args[6]);
        m_noHeapDump = Boolean.parseBoolean(args[7]);

        // arguments only used when Collector is called from VEM
        if (args.length > 8) {
            // generate resulting file in voltdbroot instead of current working dir and do not append timestamp in filename
            // so the resulting file is easier to be located and copied to VEM
            m_calledFromVEM = Boolean.parseBoolean(args[8]);

            // generate a list of information (server name, size, and path) of files rather than actually collect files
            // used by files display panel in VEM UI
            m_fileInfoOnly = Boolean.parseBoolean(args[9]);
        }

        File voltDbRoot = new File(m_voltDbRootPath);
        if (!voltDbRoot.exists()) {
            System.err.println("voltdbroot path '" + m_voltDbRootPath + "' does not exist.");
            System.exit(-1);
        }

        locatePaths(m_voltDbRootPath);

        JSONObject jsonObject = parseJSONFile(m_configInfoPath);
        parseJSONObject(jsonObject);

        List<String> collectionFilesList = listCollection(m_noHeapDump);

        if (m_dryRun) {
            System.out.println("List of the files to be collected:");
            for (String path: collectionFilesList) {
                System.out.println("  " + path);
            }
            System.out.println("[dry-run] A tgz file containing above files would be generated in current dir");
            System.out.println("          Use --upload option to enable uploading via SFTP");
        }
        else if (m_fileInfoOnly) {
            String collectionFilesListPath = m_voltDbRootPath + File.separator + m_uniqueid;

            byte jsonBytes[] = null;
            try {
                JSONStringer stringer = new JSONStringer();

                stringer.object();
                stringer.key("server").value(m_uniqueid);
                stringer.key("files").array();
                for (String path: collectionFilesList) {
                    stringer.object();
                    stringer.key("filename").value(path);
                    if (Arrays.asList(cmdFilenames).contains(path.split(" ")[0])) {
                        stringer.key("size").value(0);
                    }
                    else {
                        stringer.key("size").value(new File(path).length());
                    }
                    stringer.endObject();
                }
                stringer.endArray();
                stringer.endObject();

                JSONObject jsObj = new JSONObject(stringer.toString());
                jsonBytes = jsObj.toString(4).getBytes(Charsets.UTF_8);
            } catch (JSONException e) {
                Throwables.propagate(e);
            }

            FileOutputStream fos = null;
            try {
                fos = new FileOutputStream(collectionFilesListPath);
                fos.write(jsonBytes);
                fos.getFD().sync();
            } catch (IOException e) {
                Throwables.propagate(e);
            } finally {
                try {
                    fos.close();
                } catch (IOException e) {
                    Throwables.propagate(e);
                }
            }
        }
        else {
            generateCollection(collectionFilesList, m_calledFromVEM);
        }
    }

    private static void locatePaths(String voltDbRootPath) {
        String configLogDirPath = voltDbRootPath + File.separator + "config_log" + File.separator;

        m_configInfoPath = configLogDirPath + "config.json";
        m_catalogJarPath = configLogDirPath + "catalog.jar";
        m_deploymentPath = configLogDirPath + "deployment.xml";
    }

    private static JSONObject parseJSONFile(String configInfoPath) {
        JSONObject jsonObject = null;

        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(configInfoPath)));

            StringBuilder builder = new StringBuilder();
            String line = null;
            while ((line = bufferedReader.readLine()) != null) {
                builder.append(line);
            }
            bufferedReader.close();

            jsonObject = new JSONObject(builder.toString());
        } catch (FileNotFoundException e) {
            System.err.println("config log file '" + configInfoPath + "' could not be found.");
        } catch (IOException e) {
            System.err.println(e.getMessage());
        } catch (JSONException e) {
            System.err.println(e.getMessage());
        }

        return jsonObject;
    }

    private static void parseJSONObject(JSONObject jsonObject) {
        try {
            m_pid = jsonObject.getInt("pid");
            m_workingDir = jsonObject.getString("workingDir");

            m_logPaths.clear();
            JSONArray jsonArray = jsonObject.getJSONArray("log4jDst");
            for (int i = 0; i < jsonArray.length(); i++) {
                String path = jsonArray.getJSONObject(i).getString("path");
                m_logPaths.add(path);
            }
        } catch (JSONException e) {
            System.err.println(e.getMessage());
        }
    }

    private static List<String> listCollection(boolean noHeapDump) {
        List<String> collectionFilesList = new ArrayList<String>();

        try {
            collectionFilesList.add(m_deploymentPath);
            collectionFilesList.add(m_catalogJarPath);

            for (String path: m_logPaths) {
                for (File file: new File(path).getParentFile().listFiles()) {
                    if (file.getName().startsWith(new File(path).getName())) {
                        collectionFilesList.add(file.getCanonicalPath());
                    }
                }
            }

            for (File file: new File(m_voltDbRootPath).listFiles()) {
                if (file.getName().startsWith("voltdb_crash") && file.getName().endsWith(".txt")) {
                    collectionFilesList.add(file.getCanonicalPath());
                }
            }

            for (File file: new File(m_workingDir).listFiles()) {
                if (file.getName().startsWith("voltdb_crash") && file.getName().endsWith(".txt")) {
                    collectionFilesList.add(file.getCanonicalPath());
                }
                if (file.getName().equals("hs_err_pid" + m_pid + ".log")) {
                    collectionFilesList.add(file.getCanonicalPath());
                }
            }

            if (!noHeapDump) {
                for (File file: new File("/tmp").listFiles()) {
                    if (file.getName().equals("java_pid" + m_pid + ".hprof")) {
                        collectionFilesList.add(file.getCanonicalPath());
                    }
                }
            }

            collectionFilesList.add("sardata (result of executing \"sar -A\" if sar enabled)");
            collectionFilesList.add("dmesgdata (result of executing \"/bin/dmesg\")");

            File varlogDir = new File("/var/log");
            if (varlogDir.canRead()) {
                for (File file: varlogDir.listFiles()) {
                    if (file.getName().startsWith("syslog") || file.getName().equals("dmesg")) {
                        if (file.canRead()) {
                            collectionFilesList.add(file.getCanonicalPath());
                        }
                    }
                }
            }
        } catch (IOException e) {
            System.err.println(e.getMessage());
        }

        return collectionFilesList;
    }

    private static void generateCollection(List<String> paths, boolean calledFromVEM) {
        try {
            String timestamp = "";
            String rootpath = "";

            if (calledFromVEM) {
                rootpath = m_voltDbRootPath;
            }
            else {
                TimestampType ts = new TimestampType(new java.util.Date());
                timestamp = ts.toString().replace(' ', '-');

                // get rid of microsecond part
                timestamp = timestamp.substring(0, "YYYY-mm-DD-HH:MM:ss".length());

                rootpath = System.getProperty("user.dir");
            }

            String collectionFilePath = rootpath + File.separator + m_uniqueid + timestamp + ".tgz";
            File collectionFile = new File(collectionFilePath);
            TarGenerator tarGenerator = new TarGenerator(collectionFile, true, null);

            // Collect files with paths indicated in the list
            for (String path: paths) {
                // Skip particular items corresponding to temporary files that are only generated during collecting
                if (Arrays.asList(cmdFilenames).contains(path.split(" ")[0])) {
                    continue;
                }

                File file = new File(path);
                String filename = file.getName();

                String entryPath = file.getName();
                for (String logPath: m_logPaths) {
                    if (filename.startsWith(new File(logPath).getName())) {
                        entryPath = "log" + File.separator + file.getName();
                        break;
                    }
                }
                if (filename.startsWith("voltdb_crash")) {
                    entryPath = "voltdb_crash" + File.separator + file.getName();
                }
                if (filename.startsWith("syslog") || filename.equals("dmesg")) {
                    entryPath = "syslog" + File.separator + file.getName();
                }

                if (file.length() > 0) {
                    tarGenerator.queueEntry(entryPath, file);
                }
            }

            String[] sarCmd = {"bash", "-c", "sar -A"};
            cmd(tarGenerator, sarCmd, "sardata");

            String[] dmesgCmd = {"bash", "-c", "/bin/dmesg"};
            cmd(tarGenerator, dmesgCmd, "dmesgdata");

            tarGenerator.write(false, true);

            long sizeInByte = collectionFile.length();
            String sizeStringInKB = String.format("%5.2f", (double)sizeInByte / 1000);
            System.out.println("Collection file created at " + collectionFilePath + " size: " + sizeStringInKB + " KB");

            boolean upload = false;
            if (!m_host.isEmpty()) {
                if (m_noPrompt) {
                    upload = true;
                }
                else {
                    upload = getUserResponse("Upload via SFTP");
                }
            }

            if (upload) {
                if (org.voltdb.utils.MiscUtils.isPro()) {
                    if (m_username.isEmpty()) {
                        System.out.print("username: ");
                        m_username = System.console().readLine();
                    }
                    if (m_password.isEmpty()) {
                        System.out.print("password: ");
                        m_password = new String(System.console().readPassword());
                    }

                    try {
                        uploadToServer(collectionFilePath, m_host, m_username, m_password);

                        System.out.println("Uploaded " + new File(collectionFilePath).getName() + " via SFTP");

                        boolean delLocalCopy = false;
                        if (m_noPrompt) {
                            delLocalCopy = true;
                        }
                        else {
                            delLocalCopy = getUserResponse("Delete local copy " + collectionFilePath);
                        }

                        if (delLocalCopy) {
                            try {
                                collectionFile.delete();
                                System.out.println("Local copy "  + collectionFilePath + " deleted");
                            } catch (SecurityException e) {
                                System.out.println("Failed to delete local copy " + collectionFilePath + ". " + e.getMessage());
                            }
                        }
                    } catch (Exception e) {
                        System.err.println(e.getMessage());
                    }
                }
                else {
                    System.out.println("Uploading is only available in the Enterprise Edition");
                }
            }
        } catch (Exception e) {
            System.err.println(e.getMessage());
        }
    }

    private static boolean getUserResponse(String prompt) {
        while (true) {
            System.out.print(prompt + " [y/n]? ");
            String response = System.console().readLine();

            if (response.isEmpty()) {
                continue;
            }

            switch (response.charAt(0)) {
            case 'Y':
            case 'y':
                return true;
            case 'N':
            case 'n':
                return false;
            default:
                break;
            }
        }
    }

    private static void cmd(TarGenerator tarGenerator, String[] command, String resFilename)
            throws IOException, TarMalformatException {
        File tempFile = File.createTempFile(resFilename, null);
        tempFile.deleteOnExit();

        Process p = Runtime.getRuntime().exec(command);
        BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
        BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile));

        String line = null;
        while ((line = reader.readLine()) != null) {
            writer.write(line);
            writer.newLine();
        }
        writer.close();

        if (tempFile.length() > 0) {
            tarGenerator.queueEntry(resFilename, tempFile);
        }
    }

    public static boolean uploadToServer(String collectionFilePath, String host, String username, String password) throws Exception {
        attemptConnect(host, username, password);

        SSHTools ssh = new SSHTools(username, null);
        SFTPSession sftp = null;

        HostAndPort hostAndPort = HostAndPort.fromString(host);
        if (hostAndPort.hasPort()) {
            sftp = ssh.getSftpSession(username, password, null, hostAndPort.getHostText(), hostAndPort.getPort(), null);
        }
        else {
            sftp = ssh.getSftpSession(username, password, null, host, null);
        }

        String rootpath = sftp.exec("pwd").trim();

        HashMap<File, File> files = new HashMap<File, File>();
        File src = new File(collectionFilePath);
        File dest = new File(rootpath + File.separator + new File(collectionFilePath).getName());
        files.put(src, dest);

        try {
            sftp.ensureDirectoriesExistFor(files.values());
            sftp.copyOverFiles(files);
        } finally {
            if (sftp != null) {
                sftp.terminate();
            }
        }

        return true;
    }

    public static void attemptConnect(String host, String username, String password) throws Exception {
        SSHTools ssh = new SSHTools(username, null);

        try {
            HostAndPort hostAndPort = HostAndPort.fromString(host);
            if (hostAndPort.hasPort()) {
                ssh.getSftpSession(username, password, null, hostAndPort.getHostText(), hostAndPort.getPort(), null);
            }
            else {
                ssh.getSftpSession(username, password, null, host, null);
            }
        } catch (SFTPException e) {
            String errorMsg = e.getCause().getMessage();

            /*
             * e.getCause() is JSchException and the java exception class name only appears in message
             * hide java class name and extract error message
             */
            Pattern pattern = Pattern.compile("(?<ExceptionClass>java.*Exception: )(?<Message>.*)");
            Matcher matcher = pattern.matcher(errorMsg);

            if (matcher.matches()) {
                if (errorMsg.startsWith("java.net.UnknownHostException")) {
                    throw new Exception("Unknown host: " + matcher.group("Message"));
                }
                else {
                    throw new Exception(matcher.group("Message"));
                }
            }
            else {
                if (errorMsg.equals("Auth cancel")) {
                    // "Auth cancel" appears when username doesn't exist or password is wrong
                    throw new Exception("Authorization rejected");
                }
                else {
                    throw new Exception(errorMsg.substring(0, 1).toUpperCase() + errorMsg.substring(1));
                }
            }
        } catch (Exception e) {
            String errorMsg = e.getMessage();

            throw new Exception(errorMsg.substring(0, 1).toUpperCase() + errorMsg.substring(1));
        }
    }
}<commit_msg>fix regex matcher for error message<commit_after>/* This file is part of VoltDB.
 * Copyright (C) 2008-2013 VoltDB Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.voltdb.utils;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

import org.hsqldb_voltpatches.lib.tar.TarGenerator;
import org.hsqldb_voltpatches.lib.tar.TarMalformatException;
import org.json_voltpatches.JSONArray;
import org.json_voltpatches.JSONException;
import org.json_voltpatches.JSONObject;
import org.json_voltpatches.JSONStringer;
import org.voltdb.processtools.SFTPSession;
import org.voltdb.processtools.SFTPSession.SFTPException;
import org.voltdb.processtools.SSHTools;
import org.voltdb.types.TimestampType;

import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.net.HostAndPort;

public class Collector {
    private static String m_voltDbRootPath = null;
    private static String m_configInfoPath = null;
    private static String m_catalogJarPath = null;
    private static String m_deploymentPath = null;

    private static String m_uniqueid = "";
    private static String m_host = "";
    private static String m_username = "";
    private static String m_password = "";
    private static boolean m_noPrompt = false;
    private static boolean m_dryRun = false;
    private static boolean m_noHeapDump = false;
    private static boolean m_calledFromVEM = false;
    private static boolean m_fileInfoOnly = false;

    private static int m_pid = 0;
    private static String m_workingDir = null;
    private static List<String> m_logPaths = new ArrayList<String>();

    public static String[] cmdFilenames = {"sardata", "dmesgdata"};

    public static void main(String[] args) {
        m_voltDbRootPath = args[0];
        m_uniqueid = args[1];
        m_host = args[2];
        m_username = args[3];
        m_password = args[4];
        m_noPrompt = Boolean.parseBoolean(args[5]);
        m_dryRun = Boolean.parseBoolean(args[6]);
        m_noHeapDump = Boolean.parseBoolean(args[7]);

        // arguments only used when Collector is called from VEM
        if (args.length > 8) {
            // generate resulting file in voltdbroot instead of current working dir and do not append timestamp in filename
            // so the resulting file is easier to be located and copied to VEM
            m_calledFromVEM = Boolean.parseBoolean(args[8]);

            // generate a list of information (server name, size, and path) of files rather than actually collect files
            // used by files display panel in VEM UI
            m_fileInfoOnly = Boolean.parseBoolean(args[9]);
        }

        File voltDbRoot = new File(m_voltDbRootPath);
        if (!voltDbRoot.exists()) {
            System.err.println("voltdbroot path '" + m_voltDbRootPath + "' does not exist.");
            System.exit(-1);
        }

        locatePaths(m_voltDbRootPath);

        JSONObject jsonObject = parseJSONFile(m_configInfoPath);
        parseJSONObject(jsonObject);

        List<String> collectionFilesList = listCollection(m_noHeapDump);

        if (m_dryRun) {
            System.out.println("List of the files to be collected:");
            for (String path: collectionFilesList) {
                System.out.println("  " + path);
            }
            System.out.println("[dry-run] A tgz file containing above files would be generated in current dir");
            System.out.println("          Use --upload option to enable uploading via SFTP");
        }
        else if (m_fileInfoOnly) {
            String collectionFilesListPath = m_voltDbRootPath + File.separator + m_uniqueid;

            byte jsonBytes[] = null;
            try {
                JSONStringer stringer = new JSONStringer();

                stringer.object();
                stringer.key("server").value(m_uniqueid);
                stringer.key("files").array();
                for (String path: collectionFilesList) {
                    stringer.object();
                    stringer.key("filename").value(path);
                    if (Arrays.asList(cmdFilenames).contains(path.split(" ")[0])) {
                        stringer.key("size").value(0);
                    }
                    else {
                        stringer.key("size").value(new File(path).length());
                    }
                    stringer.endObject();
                }
                stringer.endArray();
                stringer.endObject();

                JSONObject jsObj = new JSONObject(stringer.toString());
                jsonBytes = jsObj.toString(4).getBytes(Charsets.UTF_8);
            } catch (JSONException e) {
                Throwables.propagate(e);
            }

            FileOutputStream fos = null;
            try {
                fos = new FileOutputStream(collectionFilesListPath);
                fos.write(jsonBytes);
                fos.getFD().sync();
            } catch (IOException e) {
                Throwables.propagate(e);
            } finally {
                try {
                    fos.close();
                } catch (IOException e) {
                    Throwables.propagate(e);
                }
            }
        }
        else {
            generateCollection(collectionFilesList, m_calledFromVEM);
        }
    }

    private static void locatePaths(String voltDbRootPath) {
        String configLogDirPath = voltDbRootPath + File.separator + "config_log" + File.separator;

        m_configInfoPath = configLogDirPath + "config.json";
        m_catalogJarPath = configLogDirPath + "catalog.jar";
        m_deploymentPath = configLogDirPath + "deployment.xml";
    }

    private static JSONObject parseJSONFile(String configInfoPath) {
        JSONObject jsonObject = null;

        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(configInfoPath)));

            StringBuilder builder = new StringBuilder();
            String line = null;
            while ((line = bufferedReader.readLine()) != null) {
                builder.append(line);
            }
            bufferedReader.close();

            jsonObject = new JSONObject(builder.toString());
        } catch (FileNotFoundException e) {
            System.err.println("config log file '" + configInfoPath + "' could not be found.");
        } catch (IOException e) {
            System.err.println(e.getMessage());
        } catch (JSONException e) {
            System.err.println(e.getMessage());
        }

        return jsonObject;
    }

    private static void parseJSONObject(JSONObject jsonObject) {
        try {
            m_pid = jsonObject.getInt("pid");
            m_workingDir = jsonObject.getString("workingDir");

            m_logPaths.clear();
            JSONArray jsonArray = jsonObject.getJSONArray("log4jDst");
            for (int i = 0; i < jsonArray.length(); i++) {
                String path = jsonArray.getJSONObject(i).getString("path");
                m_logPaths.add(path);
            }
        } catch (JSONException e) {
            System.err.println(e.getMessage());
        }
    }

    private static List<String> listCollection(boolean noHeapDump) {
        List<String> collectionFilesList = new ArrayList<String>();

        try {
            collectionFilesList.add(m_deploymentPath);
            collectionFilesList.add(m_catalogJarPath);

            for (String path: m_logPaths) {
                for (File file: new File(path).getParentFile().listFiles()) {
                    if (file.getName().startsWith(new File(path).getName())) {
                        collectionFilesList.add(file.getCanonicalPath());
                    }
                }
            }

            for (File file: new File(m_voltDbRootPath).listFiles()) {
                if (file.getName().startsWith("voltdb_crash") && file.getName().endsWith(".txt")) {
                    collectionFilesList.add(file.getCanonicalPath());
                }
            }

            for (File file: new File(m_workingDir).listFiles()) {
                if (file.getName().startsWith("voltdb_crash") && file.getName().endsWith(".txt")) {
                    collectionFilesList.add(file.getCanonicalPath());
                }
                if (file.getName().equals("hs_err_pid" + m_pid + ".log")) {
                    collectionFilesList.add(file.getCanonicalPath());
                }
            }

            if (!noHeapDump) {
                for (File file: new File("/tmp").listFiles()) {
                    if (file.getName().equals("java_pid" + m_pid + ".hprof")) {
                        collectionFilesList.add(file.getCanonicalPath());
                    }
                }
            }

            collectionFilesList.add("sardata (result of executing \"sar -A\" if sar enabled)");
            collectionFilesList.add("dmesgdata (result of executing \"/bin/dmesg\")");

            File varlogDir = new File("/var/log");
            if (varlogDir.canRead()) {
                for (File file: varlogDir.listFiles()) {
                    if (file.getName().startsWith("syslog") || file.getName().equals("dmesg")) {
                        if (file.canRead()) {
                            collectionFilesList.add(file.getCanonicalPath());
                        }
                    }
                }
            }
        } catch (IOException e) {
            System.err.println(e.getMessage());
        }

        return collectionFilesList;
    }

    private static void generateCollection(List<String> paths, boolean calledFromVEM) {
        try {
            String timestamp = "";
            String rootpath = "";

            if (calledFromVEM) {
                rootpath = m_voltDbRootPath;
            }
            else {
                TimestampType ts = new TimestampType(new java.util.Date());
                timestamp = ts.toString().replace(' ', '-');

                // get rid of microsecond part
                timestamp = timestamp.substring(0, "YYYY-mm-DD-HH:MM:ss".length());

                rootpath = System.getProperty("user.dir");
            }

            String collectionFilePath = rootpath + File.separator + m_uniqueid + timestamp + ".tgz";
            File collectionFile = new File(collectionFilePath);
            TarGenerator tarGenerator = new TarGenerator(collectionFile, true, null);

            // Collect files with paths indicated in the list
            for (String path: paths) {
                // Skip particular items corresponding to temporary files that are only generated during collecting
                if (Arrays.asList(cmdFilenames).contains(path.split(" ")[0])) {
                    continue;
                }

                File file = new File(path);
                String filename = file.getName();

                String entryPath = file.getName();
                for (String logPath: m_logPaths) {
                    if (filename.startsWith(new File(logPath).getName())) {
                        entryPath = "log" + File.separator + file.getName();
                        break;
                    }
                }
                if (filename.startsWith("voltdb_crash")) {
                    entryPath = "voltdb_crash" + File.separator + file.getName();
                }
                if (filename.startsWith("syslog") || filename.equals("dmesg")) {
                    entryPath = "syslog" + File.separator + file.getName();
                }

                if (file.length() > 0) {
                    tarGenerator.queueEntry(entryPath, file);
                }
            }

            String[] sarCmd = {"bash", "-c", "sar -A"};
            cmd(tarGenerator, sarCmd, "sardata");

            String[] dmesgCmd = {"bash", "-c", "/bin/dmesg"};
            cmd(tarGenerator, dmesgCmd, "dmesgdata");

            tarGenerator.write(false, true);

            long sizeInByte = collectionFile.length();
            String sizeStringInKB = String.format("%5.2f", (double)sizeInByte / 1000);
            System.out.println("Collection file created at " + collectionFilePath + " size: " + sizeStringInKB + " KB");

            boolean upload = false;
            if (!m_host.isEmpty()) {
                if (m_noPrompt) {
                    upload = true;
                }
                else {
                    upload = getUserResponse("Upload via SFTP");
                }
            }

            if (upload) {
                if (org.voltdb.utils.MiscUtils.isPro()) {
                    if (m_username.isEmpty()) {
                        System.out.print("username: ");
                        m_username = System.console().readLine();
                    }
                    if (m_password.isEmpty()) {
                        System.out.print("password: ");
                        m_password = new String(System.console().readPassword());
                    }

                    try {
                        uploadToServer(collectionFilePath, m_host, m_username, m_password);

                        System.out.println("Uploaded " + new File(collectionFilePath).getName() + " via SFTP");

                        boolean delLocalCopy = false;
                        if (m_noPrompt) {
                            delLocalCopy = true;
                        }
                        else {
                            delLocalCopy = getUserResponse("Delete local copy " + collectionFilePath);
                        }

                        if (delLocalCopy) {
                            try {
                                collectionFile.delete();
                                System.out.println("Local copy "  + collectionFilePath + " deleted");
                            } catch (SecurityException e) {
                                System.out.println("Failed to delete local copy " + collectionFilePath + ". " + e.getMessage());
                            }
                        }
                    } catch (Exception e) {
                        System.err.println(e.getMessage());
                    }
                }
                else {
                    System.out.println("Uploading is only available in the Enterprise Edition");
                }
            }
        } catch (Exception e) {
            System.err.println(e.getMessage());
        }
    }

    private static boolean getUserResponse(String prompt) {
        while (true) {
            System.out.print(prompt + " [y/n]? ");
            String response = System.console().readLine();

            if (response.isEmpty()) {
                continue;
            }

            switch (response.charAt(0)) {
            case 'Y':
            case 'y':
                return true;
            case 'N':
            case 'n':
                return false;
            default:
                break;
            }
        }
    }

    private static void cmd(TarGenerator tarGenerator, String[] command, String resFilename)
            throws IOException, TarMalformatException {
        File tempFile = File.createTempFile(resFilename, null);
        tempFile.deleteOnExit();

        Process p = Runtime.getRuntime().exec(command);
        BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
        BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile));

        String line = null;
        while ((line = reader.readLine()) != null) {
            writer.write(line);
            writer.newLine();
        }
        writer.close();

        if (tempFile.length() > 0) {
            tarGenerator.queueEntry(resFilename, tempFile);
        }
    }

    public static boolean uploadToServer(String collectionFilePath, String host, String username, String password) throws Exception {
        attemptConnect(host, username, password);

        SSHTools ssh = new SSHTools(username, null);
        SFTPSession sftp = null;

        HostAndPort hostAndPort = HostAndPort.fromString(host);
        if (hostAndPort.hasPort()) {
            sftp = ssh.getSftpSession(username, password, null, hostAndPort.getHostText(), hostAndPort.getPort(), null);
        }
        else {
            sftp = ssh.getSftpSession(username, password, null, host, null);
        }

        String rootpath = sftp.exec("pwd").trim();

        HashMap<File, File> files = new HashMap<File, File>();
        File src = new File(collectionFilePath);
        File dest = new File(rootpath + File.separator + new File(collectionFilePath).getName());
        files.put(src, dest);

        try {
            sftp.ensureDirectoriesExistFor(files.values());
            sftp.copyOverFiles(files);
        } finally {
            if (sftp != null) {
                sftp.terminate();
            }
        }

        return true;
    }

    public static void attemptConnect(String host, String username, String password) throws Exception {
        SSHTools ssh = new SSHTools(username, null);

        try {
            HostAndPort hostAndPort = HostAndPort.fromString(host);
            if (hostAndPort.hasPort()) {
                ssh.getSftpSession(username, password, null, hostAndPort.getHostText(), hostAndPort.getPort(), null);
            }
            else {
                ssh.getSftpSession(username, password, null, host, null);
            }
        } catch (SFTPException e) {
            String errorMsg = e.getCause().getMessage();

            /*
             * e.getCause() is JSchException and the java exception class name only appears in message
             * hide java class name and extract error message
             */
            Pattern pattern = Pattern.compile("(?<ExceptionClass>java.*Exception: )(?<Message>.*)");
            Matcher matcher = pattern.matcher(errorMsg);

            if (matcher.matches()) {
                if (errorMsg.startsWith("java.net.UnknownHostException")) {
                    throw new Exception("Unknown host: " + matcher.group(2));
                }
                else {
                    throw new Exception(matcher.group(2));
                }
            }
            else {
                if (errorMsg.equals("Auth cancel")) {
                    // "Auth cancel" appears when username doesn't exist or password is wrong
                    throw new Exception("Authorization rejected");
                }
                else {
                    throw new Exception(errorMsg.substring(0, 1).toUpperCase() + errorMsg.substring(1));
                }
            }
        } catch (Exception e) {
            String errorMsg = e.getMessage();

            throw new Exception(errorMsg.substring(0, 1).toUpperCase() + errorMsg.substring(1));
        }
    }
}
{'fraction_non_alphanumeric': 0.10333299564380509, 'fraction_numerical': 0.0023300577449093304, 'mean_word_length': 1.7557928531546623, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4545465', 'n_tokens_mistral': 10305, 'n_tokens_neox': 9823, 'n_words': 2369}