<commit_before>/*******************************************************************************
 *
 * Pentaho Big Data
 *
 * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.trans.steps.cassandrainput;

import java.math.BigDecimal;
import java.util.Date;

import org.apache.cassandra.db.marshal.AbstractType;

import com.datastax.driver.core.DataType;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.ColumnDefinitions;

import org.pentaho.cassandra.CassandraConnection;
import org.pentaho.cassandra.CassandraConnectionException;
import org.pentaho.cassandra.ConnectionCompression;
import org.pentaho.cassandra.Utils;
import org.pentaho.di.trans.steps.cassandrainput.CassandraInput;
import org.pentaho.di.trans.steps.cassandrainput.CassandraInputData;
import org.pentaho.di.trans.steps.cassandrainput.CassandraInputMeta;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

public class CassandraInput extends BaseStep implements StepInterface {
    private static Class<?> PKG = CassandraInput.class;

    private CassandraInputMeta meta;
    private CassandraInputData data;
    private CassandraConnection connection;

    private String[] rowColumns;

    private Boolean executeForEachInputRow = Boolean.FALSE;
    private int rowLimit;

    private String cqlStatement;

    public CassandraInput(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans) {
        super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
    }

    @Override
    public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
        this.meta = ((CassandraInputMeta) smi);
        this.data = ((CassandraInputData) sdi);
        ResultSet rs = null;

        Object[] r = getRow();
        if ((this.executeForEachInputRow) && (r == null)) {
            setOutputDone();
            return false;
        }

        if (this.first) {
            this.first = false;

            this.data.outputRowMeta = new RowMeta();
            rs = this.connection.getSession().execute(this.cqlStatement);
            this.meta.createOutputRowMeta(this.data.outputRowMeta, rs);

            this.rowColumns = this.data.outputRowMeta.getFieldNames();
        }

        try {
            int rowCount = 0;
            do {
                Object[] outputRow = new Object[this.rowColumns.length];
                Row row = rs.one();

                if (rs.isExhausted()) {
                    logDetailed("ResultSet exhausted, no rows returned.");
                    break;
                }

                incrementLinesInput();
                rowCount++;

                String[] arrayOfString;
                int j = (arrayOfString = this.rowColumns).length;

                ColumnDefinitions cd = row.getColumnDefinitions();

                for (int i = 0; i < j; i++) {
                    String col = arrayOfString[i];
                    DataType type = cd.getType(col);<commit_msg>Plugin missing last cassandra row.<commit_after>/*******************************************************************************
 *
 * Pentaho Big Data
 *
 * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.trans.steps.cassandrainput;

import java.math.BigDecimal;
import java.util.Date;

import org.apache.cassandra.db.marshal.AbstractType;

import com.datastax.driver.core.DataType;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.ColumnDefinitions;

import org.pentaho.cassandra.CassandraConnection;
import org.pentaho.cassandra.CassandraConnectionException;
import org.pentaho.cassandra.ConnectionCompression;
import org.pentaho.cassandra.Utils;
import org.pentaho.di.trans.steps.cassandrainput.CassandraInput;
import org.pentaho.di.trans.steps.cassandrainput.CassandraInputData;
import org.pentaho.di.trans.steps.cassandrainput.CassandraInputMeta;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

public class CassandraInput extends BaseStep implements StepInterface {
    private static Class<?> PKG = CassandraInput.class;

    private CassandraInputMeta meta;
    private CassandraInputData data;
    private CassandraConnection connection;

    private String[] rowColumns;

    private Boolean executeForEachInputRow = Boolean.FALSE;
    private int rowLimit;

    private String cqlStatement;

    public CassandraInput(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans) {
        super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
    }

    @Override
    public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
        this.meta = ((CassandraInputMeta) smi);
        this.data = ((CassandraInputData) sdi);
        ResultSet rs = null;

        Object[] r = getRow();
        if ((this.executeForEachInputRow) && (r == null)) {
            setOutputDone();
            return false;
        }

        if (this.first) {
            this.first = false;

            this.data.outputRowMeta = new RowMeta();
            rs = this.connection.getSession().execute(this.cqlStatement);
            this.meta.createOutputRowMeta(this.data.outputRowMeta, rs);

            this.rowColumns = this.data.outputRowMeta.getFieldNames();
        }

        try {
            int rowCount = 0;
            do {
                Object[] outputRow = new Object[this.rowColumns.length];

                if (rs.isExhausted()) {
                    logDetailed("ResultSet exhausted, no rows returned.");
                    break;
                }

                Row row = rs.one();

                incrementLinesInput();
                rowCount++;

                String[] arrayOfString;
                int j = (arrayOfString = this.rowColumns).length;

                ColumnDefinitions cd = row.getColumnDefinitions();

                for (int i = 0; i < j; i++) {
                    String col = arrayOfString[i];
                    DataType type = cd.getType(col);
{'fraction_non_alphanumeric': 0.14625769777356704, 'fraction_numerical': 0.0037896731406916154, 'mean_word_length': 3.162148841793987, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4823693', 'n_tokens_mistral': 2372, 'n_tokens_neox': 2239, 'n_words': 553}