<commit_before>package net.timbusproject.extractors.data;

import net.timbusproject.extractors.core.IExtractor;
import net.timbusproject.extractors.core.Parameter;
import net.timbusproject.extractors.core.ParameterType;
import net.timbusproject.extractors.osgi.OSGiClient;
import net.timbusproject.extractors.pojo.CallBack;
import net.timbusproject.extractors.pojo.RequestExtraction;
import net.timbusproject.extractors.pojo.RequestExtractionList;
import org.codehaus.jettison.json.JSONArray;
import org.osgi.service.log.LogService;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Set;

/**
 * Created with IntelliJ IDEA.
 * User: miguel
 * Date: 02-12-2013
 * Time: 13:30
 * To change this template use File | Settings | File Templates.
 */
public class RequestHandler {

    private Database database;

    @Qualifier("jobExplorer")
    @Autowired
    private JobExplorer explorer;

    @Qualifier("jobLauncher")
    @Autowired
    private JobLauncher launcher;

    @Qualifier("extractionJob")
    @Autowired
    private Job job;

    @Qualifier("osgiClient")
    @Autowired
    private OSGiClient client;


    @Autowired
    private LogService log;

    public RequestHandler() {
        database = new Database();
    }

    //Request extraction method. It creates new jobs according to the information provided through "extractionList"
    public long requestExtraction(RequestExtractionList extractionsList) throws JobParametersInvalidException, JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException, IOException {
        long key = database.add(extractionsList);
        database.get(key).setSemaphore();

        /*if (database.get(key).extractions.length == 1 && database.get(key).extractions[0].user == null && database.get(key).extractions[0].module != null && database.get(key).extractions[0].module.equals("Debian Software Extractor")) {
            // temporary fallback if clause
            database.get(key).extractions[0] = RequestExtraction.fromFile(getClass().getResourceAsStream("/default/debiansoftware"));
            database.get(key).extractions[0].module = "Debian Software Extractor";
        }*/


        for (RequestExtraction req : extractionsList.extractions) {
            IExtractor extractor = client.getExtractorByName(req.module);
            if (extractor == null) {
                throw new IllegalArgumentException("Extractor module does not exist");
            }
            HashMap<String, Parameter> extractorParams = extractor.getParameters();
            if (req.parameters != null) {
                for (String s : extractorParams.keySet())
                    if (!req.parameters.containsKey(s) ||
                            (extractorParams.get(s).getParameterType() == ParameterType.ARRAY && !req.parameters.get(s).startsWith("[")) ||
                            (extractorParams.get(s).getParameterType() == ParameterType.OBJECT && !req.parameters.get(s).startsWith("{"))
                            )
                        throw new IllegalArgumentException("Parameters are incorrect");
            } else
                throw new IllegalArgumentException("No parameters were sent");
            JobParametersBuilder parametersBuilder = new JobParametersBuilder()
                    .addString("wrapper", req.wrap != null ? req.wrap.toString() : null)
                    .addString("module", req.module) // selectedModule.name
                    .addLong("timestamp", System.currentTimeMillis())
                    .addLong("requestId", key);
//                    .addString("user", req.user) // newMachine (fqdn = fully qualified domain name)
//                    .addString("password", req.password != null ? "" : null)
//                    .addString("fqdn", req.fqdn) // newMachine (fqdn = fully qualified domain name)
//                    .addString("knownHosts", req.knownHosts)       //THIS
//                    .addLong("port", (long) req.port) // newMachinePort
//                    .addString("privateKey", req.privateKey) //THIS

            ArrayList<String> hiddenParameters = new ArrayList<>();
            for (String s : req.parameters.keySet()) {
                if (extractorParams.get(s).isHidden()) {
                    hiddenParameters.add(s);
                    continue;
                }
                if (extractorParams.get(s).getParameterType() == ParameterType.NUMBER)
                    parametersBuilder.addLong(s, Long.valueOf(req.parameters.get(s)));
                else
                    parametersBuilder.addString(s, req.parameters.get(s));
            }

            if (hiddenParameters.size() != 0)
                parametersBuilder.addString("hiddenFields", new JSONArray(hiddenParameters).toString());

            req.setJob(launcher.run(job, parametersBuilder.toJobParameters()));

            for (String s : hiddenParameters) {
                if (extractorParams.get(s).getParameterType() == ParameterType.NUMBER)
                    req.getJob().getExecutionContext().putLong(s, Long.valueOf(req.parameters.get(s)));
                else
                    req.getJob().getExecutionContext().putString(s, req.parameters.get(s));
                req.parameters.remove(s);
            }<commit_msg>Changed parameters testing so it accepts extra parameters. All parameters are optional now except for the module<commit_after>package net.timbusproject.extractors.data;

import net.timbusproject.extractors.core.IExtractor;
import net.timbusproject.extractors.core.Parameter;
import net.timbusproject.extractors.core.ParameterType;
import net.timbusproject.extractors.osgi.OSGiClient;
import net.timbusproject.extractors.pojo.CallBack;
import net.timbusproject.extractors.pojo.RequestExtraction;
import net.timbusproject.extractors.pojo.RequestExtractionList;
import org.codehaus.jettison.json.JSONArray;
import org.osgi.service.log.LogService;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Set;

/**
 * Created with IntelliJ IDEA.
 * User: miguel
 * Date: 02-12-2013
 * Time: 13:30
 * To change this template use File | Settings | File Templates.
 */
public class RequestHandler {

    private Database database;

    @Qualifier("jobExplorer")
    @Autowired
    private JobExplorer explorer;

    @Qualifier("jobLauncher")
    @Autowired
    private JobLauncher launcher;

    @Qualifier("extractionJob")
    @Autowired
    private Job job;

    @Qualifier("osgiClient")
    @Autowired
    private OSGiClient client;


    @Autowired
    private LogService log;

    public RequestHandler() {
        database = new Database();
    }

    //Request extraction method. It creates new jobs according to the information provided through "extractionList"
    public long requestExtraction(RequestExtractionList extractionsList) throws JobParametersInvalidException, JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException, IOException {
        long key = database.add(extractionsList);
        database.get(key).setSemaphore();

        /*if (database.get(key).extractions.length == 1 && database.get(key).extractions[0].user == null && database.get(key).extractions[0].module != null && database.get(key).extractions[0].module.equals("Debian Software Extractor")) {
            // temporary fallback if clause
            database.get(key).extractions[0] = RequestExtraction.fromFile(getClass().getResourceAsStream("/default/debiansoftware"));
            database.get(key).extractions[0].module = "Debian Software Extractor";
        }*/


        for (RequestExtraction req : extractionsList.extractions) {
            IExtractor extractor = client.getExtractorByName(req.module);
            if (extractor == null) {
                throw new IllegalArgumentException("Extractor module does not exist");
            }
            HashMap<String, Parameter> extractorParams = extractor.getParameters();
            if (req.parameters != null) {
                for (String s : extractorParams.keySet())
                    if ((extractorParams.containsKey(s) && extractorParams.get(s).getParameterType() == ParameterType.ARRAY && !req.parameters.get(s).startsWith("[")) ||
                            (extractorParams.containsKey(s) && extractorParams.get(s).getParameterType() == ParameterType.OBJECT && !req.parameters.get(s).startsWith("{"))
                            )
                        throw new IllegalArgumentException("Parameters are incorrect");
            } else
                throw new IllegalArgumentException("No parameters were sent");
            JobParametersBuilder parametersBuilder = new JobParametersBuilder()
                    .addString("wrapper", req.wrap != null ? req.wrap.toString() : null)
                    .addString("module", req.module) // selectedModule.name
                    .addLong("timestamp", System.currentTimeMillis())
                    .addLong("requestId", key);
//                    .addString("user", req.user) // newMachine (fqdn = fully qualified domain name)
//                    .addString("password", req.password != null ? "" : null)
//                    .addString("fqdn", req.fqdn) // newMachine (fqdn = fully qualified domain name)
//                    .addString("knownHosts", req.knownHosts)       //THIS
//                    .addLong("port", (long) req.port) // newMachinePort
//                    .addString("privateKey", req.privateKey) //THIS

            ArrayList<String> hiddenParameters = new ArrayList<>();
            for (String s : req.parameters.keySet()) {
                if (extractorParams.containsKey(s) && extractorParams.get(s).isHidden()) {
                    hiddenParameters.add(s);
                    continue;
                }
                if (extractorParams.containsKey(s) && extractorParams.get(s).getParameterType() == ParameterType.NUMBER)
                    parametersBuilder.addLong(s, Long.valueOf(req.parameters.get(s)));
                else
                    parametersBuilder.addString(s, req.parameters.get(s));
            }

            if (hiddenParameters.size() != 0)
                parametersBuilder.addString("hiddenFields", new JSONArray(hiddenParameters).toString());

            req.setJob(launcher.run(job, parametersBuilder.toJobParameters()));

            for (String s : hiddenParameters) {
                if (extractorParams.get(s).getParameterType() == ParameterType.NUMBER)
                    req.getJob().getExecutionContext().putLong(s, Long.valueOf(req.parameters.get(s)));
                else
                    req.getJob().getExecutionContext().putString(s, req.parameters.get(s));
                req.parameters.remove(s);
            }
{'fraction_non_alphanumeric': 0.11175843111512267, 'fraction_numerical': 0.0031180766390416018, 'mean_word_length': 2.8692063492063493, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '982207', 'n_tokens_mistral': 3194, 'n_tokens_neox': 3084, 'n_words': 593}