<commit_before>package us.kbase.mobu.tester;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringReader;
import java.io.StringWriter;
import java.net.ServerSocket;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.io.FileUtils;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.yaml.snakeyaml.Yaml;

import us.kbase.auth.AuthService;
import us.kbase.auth.AuthToken;
import us.kbase.common.executionengine.CallbackServer;
import us.kbase.common.executionengine.CallbackServerConfigBuilder;
import us.kbase.common.executionengine.LineLogger;
import us.kbase.common.executionengine.ModuleMethod;
import us.kbase.common.executionengine.ModuleRunVersion;
import us.kbase.common.executionengine.CallbackServerConfigBuilder.CallbackServerConfig;
import us.kbase.common.service.JsonServerServlet;
import us.kbase.common.service.JsonServerSyslog;
import us.kbase.common.service.UObject;
import us.kbase.mobu.util.DirUtils;
import us.kbase.mobu.util.ProcessHelper;
import us.kbase.mobu.util.TextUtils;
import us.kbase.mobu.validator.ModuleValidator;
import us.kbase.templates.TemplateFormatter;
import us.kbase.tools.WinShortPath;

public class ModuleTester {
    private File moduleDir;
    protected Map<String,Object> kbaseYmlConfig;
    private Map<String, Object> moduleContext;

    public ModuleTester() throws Exception {
        this(null);
    }
    
    public ModuleTester(File dir) throws Exception {
        moduleDir = dir == null ? DirUtils.findModuleDir() : DirUtils.findModuleDir(dir);
        String kbaseYml = TextUtils.readFileText(new File(moduleDir, "kbase.yml"));
        @SuppressWarnings("unchecked")
        Map<String,Object> config = (Map<String, Object>)new Yaml().load(kbaseYml);
        kbaseYmlConfig = config;
        moduleContext = new HashMap<String, Object>();
        moduleContext.put("module_name", kbaseYmlConfig.get("module-name"));
        moduleContext.put("module_root_path", moduleDir.getAbsolutePath());
        if (kbaseYmlConfig.get("data-version") != null) {
            moduleContext.put("data_version", kbaseYmlConfig.get("data-version"));
        }
        moduleContext.put("os_name", System.getProperty("os.name"));
    }
    
    private static void checkIgnoreLine(File f, String line) throws IOException {
        List<String> lines = new ArrayList<String>();
        if (f.exists())
            lines.addAll(FileUtils.readLines(f));
        if (!new HashSet<String>(lines).contains(line)) {
            System.out.println("Warning: file \"" + f.getName() + "\" doesn't contain \"" + line + "\" line, it will be added.");
            lines.add(line);
            FileUtils.writeLines(f, lines);
        }
    }
    
    public int runTests(String methodStoreUrl, boolean skipValidation, boolean allowSyncMethods) throws Exception {
        if (skipValidation) {
            System.out.println("Validation step is skipped");
        } else {
            ModuleValidator mv = new ModuleValidator(Arrays.asList(moduleDir.getCanonicalPath()), 
                    false, methodStoreUrl, allowSyncMethods);
            int returnCode = mv.validateAll();
            if (returnCode!=0) {
                System.out.println("You can skip validation step using -s (or --skip_validation) flag");
                System.exit(returnCode);
            }
        }
        String testLocal = "test_local";
        checkIgnoreLine(new File(moduleDir, ".gitignore"), testLocal);
        checkIgnoreLine(new File(moduleDir, ".dockerignore"), testLocal);
        File tlDir = new File(moduleDir, testLocal);
        File readmeFile = new File(tlDir, "readme.txt");
        File testCfg = new File(tlDir, "test.cfg");
        File runTestsSh = new File(tlDir, "run_tests.sh");
        File runBashSh = new File(tlDir, "run_bash.sh");
        File runDockerSh = new File(tlDir, "run_docker.sh");
        if (!tlDir.exists())
            tlDir.mkdir();
        if (!readmeFile.exists())
            TemplateFormatter.formatTemplate("module_readme_test_local", moduleContext, true, readmeFile);
        if (kbaseYmlConfig.get("data-version") != null) {
            File refDataDir = new File(tlDir, "refdata");
            if (!refDataDir.exists()) {
                TemplateFormatter.formatTemplate("module_run_tests", moduleContext, true, runTestsSh);
                refDataDir.mkdir();
            }
        }
        if (!runTestsSh.exists())
            TemplateFormatter.formatTemplate("module_run_tests", moduleContext, true, runTestsSh);
        if (!runBashSh.exists())
            TemplateFormatter.formatTemplate("module_run_bash", moduleContext, true, runBashSh);
        if (!runDockerSh.exists())
            TemplateFormatter.formatTemplate("module_run_docker", moduleContext, true, runDockerSh);
        if (!testCfg.exists()) {
            TemplateFormatter.formatTemplate("module_test_cfg", moduleContext, true, testCfg);
            System.out.println("Set KBase account credentials in test_local/test.cfg and then test again");
            return 1;
        }
        Properties props = new Properties();
        InputStream is = new FileInputStream(new File(tlDir, "test.cfg"));
        try {
            props.load(is);
        } finally {
            is.close();
        }
        String user = props.getProperty("test_user");
        String password = props.getProperty("test_password");
        if (user == null || user.trim().isEmpty()) {
            throw new IllegalStateException("Error: KBase account credentials are not set in test_local/test.cfg");
        }
        if (password == null || password.isEmpty()) {
            System.out.println("You haven't preset your password in test_local/test.cfg file. Please enter it now.");
            password = new String(System.console().readPassword("Password: "));
        }
        String token = AuthService.login(user, password).getTokenString();
        File workDir = new File(tlDir, "workdir");
        workDir.mkdir();
        File tokenFile = new File(workDir, "token");
        FileWriter fw = new FileWriter(tokenFile);
        try {
            fw.write(token);
        } finally {
            fw.close();
        }
        String endPoint = props.getProperty("kbase_endpoint");
        if (endPoint == null)
            throw new IllegalStateException("Error: KBase services end-point is not set in test_local/test.cfg");
        String jobSrvUrl = props.getProperty("job_service_url");
        if (jobSrvUrl == null)
            jobSrvUrl = endPoint + "/userandjobstate";
        String wsUrl = props.getProperty("workspace_url");
        if (wsUrl == null)
            wsUrl = endPoint + "/ws";
        String shockUrl = props.getProperty("shock_url");
        if (shockUrl == null)
            shockUrl = endPoint + "/shock-api";
        File configPropsFile = new File(workDir, "config.properties");
        PrintWriter pw = new PrintWriter(configPropsFile);
        try {
            pw.println("[global]");
            pw.println("job_service_url = " + jobSrvUrl);
            pw.println("workspace_url = " + wsUrl);
            pw.println("shock_url = " + shockUrl);<commit_msg>Trim user & pwd before login<commit_after>package us.kbase.mobu.tester;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringReader;
import java.io.StringWriter;
import java.net.ServerSocket;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.io.FileUtils;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.yaml.snakeyaml.Yaml;

import us.kbase.auth.AuthService;
import us.kbase.auth.AuthToken;
import us.kbase.common.executionengine.CallbackServer;
import us.kbase.common.executionengine.CallbackServerConfigBuilder;
import us.kbase.common.executionengine.LineLogger;
import us.kbase.common.executionengine.ModuleMethod;
import us.kbase.common.executionengine.ModuleRunVersion;
import us.kbase.common.executionengine.CallbackServerConfigBuilder.CallbackServerConfig;
import us.kbase.common.service.JsonServerServlet;
import us.kbase.common.service.JsonServerSyslog;
import us.kbase.common.service.UObject;
import us.kbase.mobu.util.DirUtils;
import us.kbase.mobu.util.ProcessHelper;
import us.kbase.mobu.util.TextUtils;
import us.kbase.mobu.validator.ModuleValidator;
import us.kbase.templates.TemplateFormatter;
import us.kbase.tools.WinShortPath;

public class ModuleTester {
    private File moduleDir;
    protected Map<String,Object> kbaseYmlConfig;
    private Map<String, Object> moduleContext;

    public ModuleTester() throws Exception {
        this(null);
    }
    
    public ModuleTester(File dir) throws Exception {
        moduleDir = dir == null ? DirUtils.findModuleDir() : DirUtils.findModuleDir(dir);
        String kbaseYml = TextUtils.readFileText(new File(moduleDir, "kbase.yml"));
        @SuppressWarnings("unchecked")
        Map<String,Object> config = (Map<String, Object>)new Yaml().load(kbaseYml);
        kbaseYmlConfig = config;
        moduleContext = new HashMap<String, Object>();
        moduleContext.put("module_name", kbaseYmlConfig.get("module-name"));
        moduleContext.put("module_root_path", moduleDir.getAbsolutePath());
        if (kbaseYmlConfig.get("data-version") != null) {
            moduleContext.put("data_version", kbaseYmlConfig.get("data-version"));
        }
        moduleContext.put("os_name", System.getProperty("os.name"));
    }
    
    private static void checkIgnoreLine(File f, String line) throws IOException {
        List<String> lines = new ArrayList<String>();
        if (f.exists())
            lines.addAll(FileUtils.readLines(f));
        if (!new HashSet<String>(lines).contains(line)) {
            System.out.println("Warning: file \"" + f.getName() + "\" doesn't contain \"" + line + "\" line, it will be added.");
            lines.add(line);
            FileUtils.writeLines(f, lines);
        }
    }
    
    public int runTests(String methodStoreUrl, boolean skipValidation, boolean allowSyncMethods) throws Exception {
        if (skipValidation) {
            System.out.println("Validation step is skipped");
        } else {
            ModuleValidator mv = new ModuleValidator(Arrays.asList(moduleDir.getCanonicalPath()), 
                    false, methodStoreUrl, allowSyncMethods);
            int returnCode = mv.validateAll();
            if (returnCode!=0) {
                System.out.println("You can skip validation step using -s (or --skip_validation) flag");
                System.exit(returnCode);
            }
        }
        String testLocal = "test_local";
        checkIgnoreLine(new File(moduleDir, ".gitignore"), testLocal);
        checkIgnoreLine(new File(moduleDir, ".dockerignore"), testLocal);
        File tlDir = new File(moduleDir, testLocal);
        File readmeFile = new File(tlDir, "readme.txt");
        File testCfg = new File(tlDir, "test.cfg");
        File runTestsSh = new File(tlDir, "run_tests.sh");
        File runBashSh = new File(tlDir, "run_bash.sh");
        File runDockerSh = new File(tlDir, "run_docker.sh");
        if (!tlDir.exists())
            tlDir.mkdir();
        if (!readmeFile.exists())
            TemplateFormatter.formatTemplate("module_readme_test_local", moduleContext, true, readmeFile);
        if (kbaseYmlConfig.get("data-version") != null) {
            File refDataDir = new File(tlDir, "refdata");
            if (!refDataDir.exists()) {
                TemplateFormatter.formatTemplate("module_run_tests", moduleContext, true, runTestsSh);
                refDataDir.mkdir();
            }
        }
        if (!runTestsSh.exists())
            TemplateFormatter.formatTemplate("module_run_tests", moduleContext, true, runTestsSh);
        if (!runBashSh.exists())
            TemplateFormatter.formatTemplate("module_run_bash", moduleContext, true, runBashSh);
        if (!runDockerSh.exists())
            TemplateFormatter.formatTemplate("module_run_docker", moduleContext, true, runDockerSh);
        if (!testCfg.exists()) {
            TemplateFormatter.formatTemplate("module_test_cfg", moduleContext, true, testCfg);
            System.out.println("Set KBase account credentials in test_local/test.cfg and then test again");
            return 1;
        }
        Properties props = new Properties();
        InputStream is = new FileInputStream(new File(tlDir, "test.cfg"));
        try {
            props.load(is);
        } finally {
            is.close();
        }
        String user = props.getProperty("test_user");
        String password = props.getProperty("test_password");
        if (user == null || user.trim().isEmpty()) {
            throw new IllegalStateException("Error: KBase account credentials are not set in test_local/test.cfg");
        }
        if (password == null || password.trim().isEmpty()) {
            System.out.println("You haven't preset your password in test_local/test.cfg file. Please enter it now.");
            password = new String(System.console().readPassword("Password: "));
        }
        String token = AuthService.login(user.trim(), password.trim())
                .getTokenString();
        File workDir = new File(tlDir, "workdir");
        workDir.mkdir();
        File tokenFile = new File(workDir, "token");
        FileWriter fw = new FileWriter(tokenFile);
        try {
            fw.write(token);
        } finally {
            fw.close();
        }
        String endPoint = props.getProperty("kbase_endpoint");
        if (endPoint == null)
            throw new IllegalStateException("Error: KBase services end-point is not set in test_local/test.cfg");
        String jobSrvUrl = props.getProperty("job_service_url");
        if (jobSrvUrl == null)
            jobSrvUrl = endPoint + "/userandjobstate";
        String wsUrl = props.getProperty("workspace_url");
        if (wsUrl == null)
            wsUrl = endPoint + "/ws";
        String shockUrl = props.getProperty("shock_url");
        if (shockUrl == null)
            shockUrl = endPoint + "/shock-api";
        File configPropsFile = new File(workDir, "config.properties");
        PrintWriter pw = new PrintWriter(configPropsFile);
        try {
            pw.println("[global]");
            pw.println("job_service_url = " + jobSrvUrl);
            pw.println("workspace_url = " + wsUrl);
            pw.println("shock_url = " + shockUrl);
{'fraction_non_alphanumeric': 0.11916179465282796, 'fraction_numerical': 0.0002627602969191355, 'mean_word_length': 2.9707876890975484, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6952813', 'n_tokens_mistral': 4318, 'n_tokens_neox': 4261, 'n_words': 908}