<commit_before>/**
 * Copyright 2013 Unicon (R) Licensed under the
 * Educational Community License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 * http://www.osedu.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS"
 * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package org.apereo.lap.services.pipeline;

import org.apache.commons.lang.StringUtils;
import org.apereo.lap.model.PipelineConfig;
import org.apereo.lap.model.Processor;
import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.Result;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.util.EnvUtil;
import org.pentaho.di.scoring.WekaScoringMeta;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.steps.jsoninput.JsonInputMeta;
import org.pentaho.di.trans.steps.jsonoutput.JsonOutputMeta;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;

import java.io.File;
import java.util.List;

/**
 * Handles the pipeline processing for Kettle processors
 *
 * @author <NAME> (azeckoski @ unicon.net) (azeckoski @ vt.edu)
 * @author <NAME> (rlong @ unicon.net)
 */
@Component
public class KettleTransformPipelineProcessor extends KettleBasePipelineProcessor {

    private String jsonInputFilename = "sample1_input.json";
    private String jsonOutputFilename = "sample1_output.json";
    private String scoringModelFilename = "/kettle/Marist_OAAI_ACADEMIC_RISK.xml";

    @PostConstruct
    public void init() {
        setKettlePluginsDirectory();
    }

    @Override
    public Processor.ProcessorType getProcessorType() {
        return Processor.ProcessorType.KETTLE_TRANSFORM;
    }

    @Override
    public ProcessorResult process(PipelineConfig pipelineConfig, Processor processorConfig) {
        ProcessorResult result = new ProcessorResult(Processor.ProcessorType.KETTLE_TRANSFORM);
        File kettleXMLFile = getFile(processorConfig.filename);

        try {
            KettleEnvironment.init(false);
            EnvUtil.environmentInit();
            TransMeta transMeta = new TransMeta(kettleXMLFile.getAbsolutePath());

            // update the database connections to use the temporary in-memory database
            List<DatabaseMeta> databaseMetas = transMeta.getDatabases();
            for (DatabaseMeta databaseMeta : databaseMetas) {
                updateDatabaseConnection(databaseMeta);
            }
<commit_msg>LAI-5: updated error handling and comments<commit_after>/**
 * Copyright 2013 Unicon (R) Licensed under the
 * Educational Community License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 * http://www.osedu.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS"
 * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package org.apereo.lap.services.pipeline;

import org.apache.commons.lang.StringUtils;
import org.apereo.lap.model.PipelineConfig;
import org.apereo.lap.model.Processor;
import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.Result;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.util.EnvUtil;
import org.pentaho.di.scoring.WekaScoringMeta;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.steps.jsoninput.JsonInputMeta;
import org.pentaho.di.trans.steps.jsonoutput.JsonOutputMeta;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;

import java.io.File;
import java.util.List;

/**
 * Handles the pipeline processing for Kettle processors
 *
 * @author <NAME> (azeckoski @ unicon.net) (azeckoski @ vt.edu)
 * @author <NAME> (rlong @ unicon.net)
 */
@Component
public class KettleTransformPipelineProcessor extends KettleBasePipelineProcessor {

    private String jsonInputFilename = "sample1_input.json";
    private String jsonOutputFilename = "sample1_output.json";
    private String scoringModelFilename = "/kettle/Marist_OAAI_ACADEMIC_RISK.xml";

    @PostConstruct
    public void init() {
        // Do any init here you need to (but note this is for the service and not each run)
        setKettlePluginsDirectory();
    }

    @Override
    public Processor.ProcessorType getProcessorType() {
        return Processor.ProcessorType.KETTLE_TRANSFORM;
    }

    @Override
    public ProcessorResult process(PipelineConfig pipelineConfig, Processor processorConfig) {
        ProcessorResult result = new ProcessorResult(Processor.ProcessorType.KETTLE_TRANSFORM);
        File kettleXMLFile = getFile(processorConfig.filename);

        try {
            KettleEnvironment.init(false);
            EnvUtil.environmentInit();
            TransMeta transMeta = new TransMeta(kettleXMLFile.getAbsolutePath());

            // update the database connections to use the temporary in-memory database
            List<DatabaseMeta> databaseMetas = transMeta.getDatabases();
            for (DatabaseMeta databaseMeta : databaseMetas) {
                updateDatabaseConnection(databaseMeta);
            }

{'fraction_non_alphanumeric': 0.0873634945397816, 'fraction_numerical': 0.003640145605824233, 'mean_word_length': 3.940068493150685, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '423141', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1586, 'n_words': 433}