<commit_before>/* This file is part of VoltDB.
 * Copyright (C) 2008-2014 VoltDB Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.voltdb.utils;

import com.google_voltpatches.common.collect.Lists;
import org.voltcore.logging.VoltLogger;
import org.voltdb.VoltTable;
import org.voltdb.VoltType;
import org.voltdb.client.Client;
import org.voltdb.client.ClientImpl;
import org.voltdb.client.ClientResponse;
import org.voltdb.client.NoConnectionsException;
import org.voltdb.client.ProcCallException;
import org.voltdb.client.ProcedureCallback;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;

/**
 * A CSVDataLoader implementation that inserts one row at a time.
 */
public class CSVTupleDataLoader implements CSVDataLoader {
    private static final VoltLogger m_log = new VoltLogger("CSVLOADER");

    private final Client m_client;
    private final String m_insertProcedure;
    private final VoltType[] m_columnTypes;
    private final CSVLoaderErrorHandler m_errHandler;

    final AtomicLong m_processedCount = new AtomicLong(0);
    final AtomicLong m_failedCount = new AtomicLong(0);
    final int m_reportEveryNRows = 10000;

    @Override
    public void setFlushInterval(int delay, int seconds) {
        //no op
    }

    //Callback for single row procedure invoke called for rows in failed batch.
    private class PartitionSingleExecuteProcedureCallback implements ProcedureCallback {
        final CSVLineWithMetaData m_csvLine;

        public PartitionSingleExecuteProcedureCallback(CSVLineWithMetaData csvLine) {
            m_csvLine = csvLine;
        }

        //one insert at a time callback
        @Override
        public void clientCallback(ClientResponse response) throws Exception {
            byte status = response.getStatus();
            if (status != ClientResponse.SUCCESS) {
                m_failedCount.incrementAndGet();
                m_errHandler.handleError(m_csvLine, response, response.getStatusString());
            }
            long currentCount = m_processedCount.incrementAndGet();

            if (currentCount % m_reportEveryNRows == 0) {
                m_log.info("Inserted " + currentCount + " rows");
            }
        }
    }

    public CSVTupleDataLoader(ClientImpl client, String procName, CSVLoaderErrorHandler errHandler)
            throws IOException, ProcCallException
    {
        m_client = client;
        m_insertProcedure = procName;
        m_errHandler = errHandler;

        List<VoltType> typeList = Lists.newArrayList();
        VoltTable procInfo = client.callProcedure("@SystemCatalog", "PROCEDURECOLUMNS").getResults()[0];
        while (procInfo.advanceRow()) {
            if (m_insertProcedure.equalsIgnoreCase(procInfo.getString("PROCEDURE_NAME"))) {
                String typeStr = (String) procInfo.get("TYPE_NAME", VoltType.STRING);
                typeList.add(VoltType.typeFromString(typeStr));
            }
        }
        if (typeList.isEmpty()) {
            //csvloader will exit
            throw new RuntimeException("No matching insert procedure available");
        }<commit_msg>ENG-6344: Fix miscounting in -p option.<commit_after>/* This file is part of VoltDB.
 * Copyright (C) 2008-2014 VoltDB Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.voltdb.utils;

import com.google_voltpatches.common.collect.Lists;
import org.voltcore.logging.VoltLogger;
import org.voltdb.VoltTable;
import org.voltdb.VoltType;
import org.voltdb.client.Client;
import org.voltdb.client.ClientImpl;
import org.voltdb.client.ClientResponse;
import org.voltdb.client.NoConnectionsException;
import org.voltdb.client.ProcCallException;
import org.voltdb.client.ProcedureCallback;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;

/**
 * A CSVDataLoader implementation that inserts one row at a time.
 */
public class CSVTupleDataLoader implements CSVDataLoader {
    private static final VoltLogger m_log = new VoltLogger("CSVLOADER");

    private final Client m_client;
    private final String m_insertProcedure;
    private final VoltType[] m_columnTypes;
    private final CSVLoaderErrorHandler m_errHandler;

    final AtomicLong m_processedCount = new AtomicLong(0);
    final AtomicLong m_failedCount = new AtomicLong(0);
    final int m_reportEveryNRows = 10000;

    @Override
    public void setFlushInterval(int delay, int seconds) {
        //no op
    }

    //Callback for single row procedure invoke called for rows in failed batch.
    private class PartitionSingleExecuteProcedureCallback implements ProcedureCallback {
        final CSVLineWithMetaData m_csvLine;

        public PartitionSingleExecuteProcedureCallback(CSVLineWithMetaData csvLine) {
            m_csvLine = csvLine;
        }

        //one insert at a time callback
        @Override
        public void clientCallback(ClientResponse response) throws Exception {
            byte status = response.getStatus();
            if (status != ClientResponse.SUCCESS) {
                m_failedCount.incrementAndGet();
                m_errHandler.handleError(m_csvLine, response, response.getStatusString());
            }
            long currentCount = m_processedCount.incrementAndGet();

            if (currentCount % m_reportEveryNRows == 0) {
                m_log.info("Inserted " + (currentCount - m_failedCount.get()) + " rows");
            }
        }
    }

    public CSVTupleDataLoader(ClientImpl client, String procName, CSVLoaderErrorHandler errHandler)
            throws IOException, ProcCallException
    {
        m_client = client;
        m_insertProcedure = procName;
        m_errHandler = errHandler;

        List<VoltType> typeList = Lists.newArrayList();
        VoltTable procInfo = client.callProcedure("@SystemCatalog", "PROCEDURECOLUMNS").getResults()[0];
        while (procInfo.advanceRow()) {
            if (m_insertProcedure.equalsIgnoreCase(procInfo.getString("PROCEDURE_NAME"))) {
                String typeStr = (String) procInfo.get("TYPE_NAME", VoltType.STRING);
                typeList.add(VoltType.typeFromString(typeStr));
            }
        }
        if (typeList.isEmpty()) {
            //csvloader will exit
            throw new RuntimeException("No matching insert procedure available");
        }
{'fraction_non_alphanumeric': 0.07819905213270142, 'fraction_numerical': 0.0052659294365455505, 'mean_word_length': 3.171883580450302, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5320925', 'n_tokens_mistral': 2113, 'n_tokens_neox': 2020, 'n_words': 616}