<commit_before>/**
 * Copyright 2013 Unicon (R) Licensed under the
 * Educational Community License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 * http://www.osedu.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS"
 * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package org.apereo.lap.services.pipeline;

import org.apereo.lap.model.PipelineConfig;
import org.apereo.lap.model.Processor;
import org.apereo.lap.services.ConfigurationService;
import org.apereo.lap.services.StorageService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;

import java.util.List;
import java.util.Map;

/**
 * This processor just produces Fake data in a "KETTLE_DATA" table
 * The table has these fields: ID (auto), ALTERNATIVE_ID, ACADEMIC_RISK
 *
 * @author <NAME> (rlong @ unicon.net)
 */
@Component
public class KettleDataPipelineProcessor implements PipelineProcessor {

    static final Logger logger = LoggerFactory.getLogger(KettleDataPipelineProcessor.class);

    @Resource
    ConfigurationService config;

    @Resource
    StorageService storage;

    @PostConstruct
    public void init() {
        // create the temp table
        storage.getTempJdbcTemplate().execute(
                "CREATE TABLE IF NOT EXISTS KETTLE_DATA (" +
                "  ID INT(11) NOT NULL AUTO_INCREMENT," +
                "  ALTERNATIVE_ID VARCHAR(255) NOT NULL," +
                "  COURSE_ID VARCHAR(255) NOT NULL," +
                "  MODEL_RISK_CONFIDENCE VARCHAR(255), " +
                "  FAIL_PROBABILITY VARCHAR(255)," +
                "  PASS_PROBABILITY VARCHAR(255)," +
                "  PRIMARY KEY (ID)," +
                "  UNIQUE KEY KETTLE_DATA_UNIQUE (ALTERNATIVE_ID, COURSE_ID)" +
                ")"
        );
        logger.info("INIT: created temp table KETTLE_DATA");
    }

    @Override
    public Processor.ProcessorType getProcessorType() {
        return Processor.ProcessorType.KETTLE_DATA;
    }

    @Override
    public ProcessorResult process(PipelineConfig pipelineConfig, Processor processorConfig, String inputJson) {
        ProcessorResult result = new ProcessorResult(Processor.ProcessorType.KETTLE_DATA);<commit_msg>LAI-5 added extra fields to the kettle output csv<commit_after>/**
 * Copyright 2013 Unicon (R) Licensed under the
 * Educational Community License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 * http://www.osedu.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS"
 * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package org.apereo.lap.services.pipeline;

import org.apereo.lap.model.PipelineConfig;
import org.apereo.lap.model.Processor;
import org.apereo.lap.services.ConfigurationService;
import org.apereo.lap.services.StorageService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;

import java.util.List;
import java.util.Map;

/**
 * This processor just produces Fake data in a "KETTLE_DATA" table
 * The table has these fields: ID (auto), ALTERNATIVE_ID, ACADEMIC_RISK
 *
 * @author <NAME> (<EMAIL>)
 */
@Component
public class KettleDataPipelineProcessor implements PipelineProcessor {

    static final Logger logger = LoggerFactory.getLogger(KettleDataPipelineProcessor.class);

    @Resource
    ConfigurationService config;

    @Resource
    StorageService storage;

    @PostConstruct
    public void init() {
        // create the temp table
        storage.getTempJdbcTemplate().execute(
                "CREATE TABLE IF NOT EXISTS KETTLE_DATA (" +
                "  ID INT(11) NOT NULL AUTO_INCREMENT," +
                "  ALTERNATIVE_ID VARCHAR(255) NOT NULL," +
                "  COURSE_ID VARCHAR(255) NOT NULL," +
                "  MODEL_RISK_CONFIDENCE VARCHAR(255), " +
                "  FAIL_PROBABILITY DECIMAL," +
                "  PASS_PROBABILITY DECIMAL," +
                "  PRIMARY KEY (ID)," +
                "  UNIQUE KEY KETTLE_DATA_UNIQUE (ALTERNATIVE_ID, COURSE_ID)" +
                ")"
        );
        logger.info("INIT: created temp table KETTLE_DATA");
    }

    @Override
    public Processor.ProcessorType getProcessorType() {
        return Processor.ProcessorType.KETTLE_DATA;
    }

    @Override
    public ProcessorResult process(PipelineConfig pipelineConfig, Processor processorConfig, String inputJson) {
        ProcessorResult result = new ProcessorResult(Processor.ProcessorType.KETTLE_DATA);
{'fraction_non_alphanumeric': 0.08493254797643929, 'fraction_numerical': 0.009310279308379252, 'mean_word_length': 3.201117318435754, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4370660', 'n_tokens_mistral': 1565, 'n_tokens_neox': 1463, 'n_words': 434}