<commit_before>/*
 * This file is part of the PSL software.
 * Copyright 2011-2015 University of Maryland
 * Copyright 2013-2022 The Regents of the University of California
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.linqs.psl.cli;

import org.linqs.psl.application.inference.InferenceApplication;
import org.linqs.psl.application.inference.online.messages.responses.OnlineResponse;
import org.linqs.psl.application.learning.weight.WeightLearningApplication;
import org.linqs.psl.config.Options;
import org.linqs.psl.config.RuntimeOptions;
import org.linqs.psl.database.DataStore;
import org.linqs.psl.database.Database;
import org.linqs.psl.database.Partition;
import org.linqs.psl.database.rdbms.RDBMSDataStore;
import org.linqs.psl.database.rdbms.driver.DatabaseDriver;
import org.linqs.psl.database.rdbms.driver.H2DatabaseDriver;
import org.linqs.psl.database.rdbms.driver.H2DatabaseDriver.Type;
import org.linqs.psl.database.rdbms.driver.PostgreSQLDriver;
import org.linqs.psl.evaluation.statistics.Evaluator;
import org.linqs.psl.grounding.GroundRuleStore;
import org.linqs.psl.model.Model;
import org.linqs.psl.model.predicate.StandardPredicate;
import org.linqs.psl.model.rule.GroundRule;
import org.linqs.psl.model.rule.Rule;
import org.linqs.psl.model.rule.UnweightedGroundRule;
import org.linqs.psl.model.rule.WeightedGroundRule;
import org.linqs.psl.parser.ModelLoader;
import org.linqs.psl.runtime.Runtime;
import org.linqs.psl.util.FileUtils;
import org.linqs.psl.util.ListUtils;
import org.linqs.psl.util.Logger;
import org.linqs.psl.util.Reflection;
import org.linqs.psl.util.Version;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.configuration2.ex.ConfigurationException;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Set;

/**
 * Launches PSL from the command line.
 * Supports inference and supervised parameter learning.
 */
public class Launcher {
    private static final Logger log = Logger.getLogger(Launcher.class);
    private CommandLine parsedOptions;

    private Launcher(CommandLine givenOptions) {
        this.parsedOptions = givenOptions;
    }

    private void outputServerResponses(List<OnlineResponse> serverResponses) {
        for (OnlineResponse response : serverResponses) {
            System.out.println(response.toString());
        }
    }

    private void outputServerResponses(List<OnlineResponse> serverResponses, String outputFilePath) {
        Path outputDirectory = Paths.get(outputFilePath).getParent();
        if (outputDirectory != null) {
            FileUtils.mkdir(outputDirectory.toString());
        }

        try (BufferedWriter bufferedWriter = FileUtils.getBufferedWriter(outputFilePath)) {
            for (OnlineResponse response : serverResponses) {
                bufferedWriter.write(response.toString() + "\n");
            }
        } catch (IOException ex) {
            throw new RuntimeException(String.format("Error writing online server responses to file: %s", outputFilePath), ex);
        }
    }

    private void runOnlineClient() {
        log.info("Starting OnlinePSL client.");
        List<OnlineResponse> serverResponses = OnlineActionInterface.run();
        log.info("OnlinePSL client closed.");

        // Output the results.
        if (!(parsedOptions.hasOption(CommandLineLoader.OPTION_ONLINE_SERVER_RESPONSE_OUTPUT))) {
            log.trace("Writing server responses to stdout.");
            outputServerResponses(serverResponses);
        } else {
            String outputFilePath = parsedOptions.getOptionValue(CommandLineLoader.OPTION_ONLINE_SERVER_RESPONSE_OUTPUT);
            log.trace("Writing inferred predicates to file: " + outputFilePath);
            outputServerResponses(serverResponses, outputFilePath);
        }
    }

    /**
     * Convert all compatible options to the PSL runtime.
     */
    private void convertRuntimeOptions() {
        boolean hasInference = false;
        boolean hasLearn = false;

        if (parsedOptions.hasOption(CommandLineLoader.OPERATION_INFER)) {
            hasInference = true;
            RuntimeOptions.INFERENCE.set(true);

            String method = parsedOptions.getOptionValue(CommandLineLoader.OPERATION_INFER);
            if (method != null) {
                RuntimeOptions.INFERENCE_METHOD.set(method);
            }
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPERATION_LEARN)) {
            hasLearn = true;
            RuntimeOptions.LEARN.set(true);

            String method = parsedOptions.getOptionValue(CommandLineLoader.OPERATION_LEARN);
            if (method != null) {
                RuntimeOptions.LEARN_METHOD.set(method);
            }
        }

        if (!hasInference && !hasLearn) {
            RuntimeOptions.INFERENCE.set(true);
        }

        // HACK(eriq): Since the CLU currently only supports one mode (infer/learn) at a time,
        // we will just set both modes when we see data/model files.

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_DATA)) {
            RuntimeOptions.INFERENCE_DATA_PATH.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_DATA));
            RuntimeOptions.LEARN_DATA_PATH.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_DATA));
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_MODEL)) {
            RuntimeOptions.INFERENCE_MODEL_PATH.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_MODEL));
            RuntimeOptions.LEARN_MODEL_PATH.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_MODEL));
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_DB_H2_PATH)) {
            RuntimeOptions.DB_H2.set(true);
            RuntimeOptions.DB_H2_PATH.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_DB_H2_PATH));
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_DB_POSTGRESQL_NAME)) {
            RuntimeOptions.DB_PG.set(true);
            RuntimeOptions.DB_PG_NAME.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_DB_POSTGRESQL_NAME));
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_EVAL)) {
            List<String> evaluatorNames = new ArrayList<String>();
            for (String evaluatorName : parsedOptions.getOptionValues(CommandLineLoader.OPTION_EVAL)) {
                evaluatorNames.add(evaluatorName);<commit_msg>Fixed an issue in the CLI/Runtime where learned models are not written out.<commit_after>/*
 * This file is part of the PSL software.
 * Copyright 2011-2015 University of Maryland
 * Copyright 2013-2022 The Regents of the University of California
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.linqs.psl.cli;

import org.linqs.psl.application.inference.InferenceApplication;
import org.linqs.psl.application.inference.online.messages.responses.OnlineResponse;
import org.linqs.psl.application.learning.weight.WeightLearningApplication;
import org.linqs.psl.config.Options;
import org.linqs.psl.config.RuntimeOptions;
import org.linqs.psl.database.DataStore;
import org.linqs.psl.database.Database;
import org.linqs.psl.database.Partition;
import org.linqs.psl.database.rdbms.RDBMSDataStore;
import org.linqs.psl.database.rdbms.driver.DatabaseDriver;
import org.linqs.psl.database.rdbms.driver.H2DatabaseDriver;
import org.linqs.psl.database.rdbms.driver.H2DatabaseDriver.Type;
import org.linqs.psl.database.rdbms.driver.PostgreSQLDriver;
import org.linqs.psl.evaluation.statistics.Evaluator;
import org.linqs.psl.grounding.GroundRuleStore;
import org.linqs.psl.model.Model;
import org.linqs.psl.model.predicate.StandardPredicate;
import org.linqs.psl.model.rule.GroundRule;
import org.linqs.psl.model.rule.Rule;
import org.linqs.psl.model.rule.UnweightedGroundRule;
import org.linqs.psl.model.rule.WeightedGroundRule;
import org.linqs.psl.parser.ModelLoader;
import org.linqs.psl.runtime.Runtime;
import org.linqs.psl.util.FileUtils;
import org.linqs.psl.util.ListUtils;
import org.linqs.psl.util.Logger;
import org.linqs.psl.util.Reflection;
import org.linqs.psl.util.Version;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.configuration2.ex.ConfigurationException;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Set;

/**
 * Launches PSL from the command line.
 * Supports inference and supervised parameter learning.
 */
public class Launcher {
    private static final Logger log = Logger.getLogger(Launcher.class);
    private CommandLine parsedOptions;

    private Launcher(CommandLine givenOptions) {
        this.parsedOptions = givenOptions;
    }

    private void outputServerResponses(List<OnlineResponse> serverResponses) {
        for (OnlineResponse response : serverResponses) {
            System.out.println(response.toString());
        }
    }

    private void outputServerResponses(List<OnlineResponse> serverResponses, String outputFilePath) {
        Path outputDirectory = Paths.get(outputFilePath).getParent();
        if (outputDirectory != null) {
            FileUtils.mkdir(outputDirectory.toString());
        }

        try (BufferedWriter bufferedWriter = FileUtils.getBufferedWriter(outputFilePath)) {
            for (OnlineResponse response : serverResponses) {
                bufferedWriter.write(response.toString() + "\n");
            }
        } catch (IOException ex) {
            throw new RuntimeException(String.format("Error writing online server responses to file: %s", outputFilePath), ex);
        }
    }

    private void runOnlineClient() {
        log.info("Starting OnlinePSL client.");
        List<OnlineResponse> serverResponses = OnlineActionInterface.run();
        log.info("OnlinePSL client closed.");

        // Output the results.
        if (!(parsedOptions.hasOption(CommandLineLoader.OPTION_ONLINE_SERVER_RESPONSE_OUTPUT))) {
            log.trace("Writing server responses to stdout.");
            outputServerResponses(serverResponses);
        } else {
            String outputFilePath = parsedOptions.getOptionValue(CommandLineLoader.OPTION_ONLINE_SERVER_RESPONSE_OUTPUT);
            log.trace("Writing inferred predicates to file: " + outputFilePath);
            outputServerResponses(serverResponses, outputFilePath);
        }
    }

    /**
     * Convert all compatible options to the PSL runtime.
     */
    private void convertRuntimeOptions() {
        boolean hasInference = false;
        boolean hasLearn = false;

        if (parsedOptions.hasOption(CommandLineLoader.OPERATION_INFER)) {
            hasInference = true;
            RuntimeOptions.INFERENCE.set(true);

            String method = parsedOptions.getOptionValue(CommandLineLoader.OPERATION_INFER);
            if (method != null) {
                RuntimeOptions.INFERENCE_METHOD.set(method);
            }
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPERATION_LEARN)) {
            hasLearn = true;
            RuntimeOptions.LEARN.set(true);

            String method = parsedOptions.getOptionValue(CommandLineLoader.OPERATION_LEARN);
            if (method != null) {
                RuntimeOptions.LEARN_METHOD.set(method);
            }
        }

        if (!hasInference && !hasLearn) {
            RuntimeOptions.INFERENCE.set(true);
        }

        // HACK(eriq): Since the CLI currently only supports one mode (infer/learn) at a time,
        // we will just set both modes when we see data/model files.

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_DATA)) {
            RuntimeOptions.INFERENCE_DATA_PATH.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_DATA));
            RuntimeOptions.LEARN_DATA_PATH.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_DATA));
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_MODEL)) {
            String modelPath = parsedOptions.getOptionValue(CommandLineLoader.OPTION_MODEL);

            RuntimeOptions.INFERENCE_MODEL_PATH.set(modelPath);
            RuntimeOptions.LEARN_MODEL_PATH.set(modelPath);

            RuntimeOptions.LEARN_OUTPUT_MODEL_PATH.set(modelPath.replaceFirst("\\.psl$", "-learned.psl"));
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_DB_H2_PATH)) {
            RuntimeOptions.DB_H2.set(true);
            RuntimeOptions.DB_H2_PATH.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_DB_H2_PATH));
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_DB_POSTGRESQL_NAME)) {
            RuntimeOptions.DB_PG.set(true);
            RuntimeOptions.DB_PG_NAME.set(parsedOptions.getOptionValue(CommandLineLoader.OPTION_DB_POSTGRESQL_NAME));
        }

        if (parsedOptions.hasOption(CommandLineLoader.OPTION_EVAL)) {
            List<String> evaluatorNames = new ArrayList<String>();
            for (String evaluatorName : parsedOptions.getOptionValues(CommandLineLoader.OPTION_EVAL)) {
                evaluatorNames.add(evaluatorName);
{'fraction_non_alphanumeric': 0.09402944008804512, 'fraction_numerical': 0.003714403631861329, 'mean_word_length': 3.480431432973806, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1457231', 'n_tokens_mistral': 4217, 'n_tokens_neox': 4130, 'n_words': 804}