<commit_before>package org.apache.cassandra.hadoop;

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOError;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import org.apache.cassandra.EmbeddedServer;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.thrift.Cassandra;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.ColumnOrSuperColumn;
import org.apache.cassandra.thrift.ColumnParent;
import org.apache.cassandra.thrift.ConsistencyLevel;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.thrift.KeyRange;
import org.apache.cassandra.thrift.KeySlice;
import org.apache.cassandra.thrift.KsDef;
import org.apache.cassandra.thrift.SlicePredicate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

/**
 * A test case for the {@link ColumnFamilyOutputFormat}, which reads each
 * &lt;key, value&gt; pair from a sequence file, maps them to a &lt;key,
 * column&gt; pair, and then reduces it by aggregating the columns that
 * correspond to the same key. Finally, the output &lt;key, columns&gt; pairs
 * are written into the (Cassandra) column family associated with this
 * {@link OutputFormat}.
 * 
 * @author <NAME>
 * 
 */

public class ColumnFamilyOutputFormatTest extends EmbeddedServer
{
    static final String KEYSPACE = "ColumnFamilyOutputFormatTestKeyspace";
    static final String COLUMN_FAMILY = "outputColumnFamily";

    private static final String INPUT_FOLDER = "columnfamily.outputtest";

    private static final String INPUT_FILE = "rows.txt";

    private static final int NUMBER_OF_ROWS = 3;
    private static final int NUMBER_OF_COLUMNS = 4;

    List<Integer> rowKeys;
    List<Integer> columnValues;

    private static FileSystem fs;

    static
    {
        try
        {
            fs = FileSystem.getLocal(new Configuration());
        }
        catch (IOException ioe)
        {
            throw new IOError(ioe);
        }
    }

    private Cassandra.Client thriftClient;

    @Before
    public void setup() throws Exception
    {
        setupCassandra();
        defineRows();
        createInputFile();
    }

    @After
    public void tearDown() throws Exception
    {
        deleteInputFile();
        deleteRows();
    }

    private void deleteRows() throws InvalidRequestException, TException
    {
        thriftClient.system_drop_column_family(COLUMN_FAMILY);
    }

    private void deleteInputFile() throws IOException
    {
        inputdir = new Path(INPUT_FOLDER);
        if (!fs.delete(inputdir, true))
            throw new IOException("Delete failed to remove " + inputdir.toString());
    }

    private void setupCassandra() throws TException, InvalidRequestException
    {
        /* Establish a thrift connection to the cassandra instance */
        TSocket socket = new TSocket(DatabaseDescriptor.getListenAddress().getHostName(), DatabaseDescriptor.getRpcPort());
        TTransport transport;
        transport = socket;
        TBinaryProtocol binaryProtocol = new TBinaryProtocol(transport, false, false);
        Cassandra.Client cassandraClient = new Cassandra.Client(binaryProtocol);
        transport.open();<commit_msg>switch CFOutputFormatTest client to Framed mode.  patch by jbellis<commit_after>package org.apache.cassandra.hadoop;

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOError;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import org.apache.cassandra.EmbeddedServer;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.thrift.Cassandra;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.ColumnOrSuperColumn;
import org.apache.cassandra.thrift.ColumnParent;
import org.apache.cassandra.thrift.ConsistencyLevel;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.thrift.KeyRange;
import org.apache.cassandra.thrift.KeySlice;
import org.apache.cassandra.thrift.KsDef;
import org.apache.cassandra.thrift.SlicePredicate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

/**
 * A test case for the {@link ColumnFamilyOutputFormat}, which reads each
 * &lt;key, value&gt; pair from a sequence file, maps them to a &lt;key,
 * column&gt; pair, and then reduces it by aggregating the columns that
 * correspond to the same key. Finally, the output &lt;key, columns&gt; pairs
 * are written into the (Cassandra) column family associated with this
 * {@link OutputFormat}.
 * 
 * @author <NAME>
 * 
 */

public class ColumnFamilyOutputFormatTest extends EmbeddedServer
{
    static final String KEYSPACE = "ColumnFamilyOutputFormatTestKeyspace";
    static final String COLUMN_FAMILY = "outputColumnFamily";

    private static final String INPUT_FOLDER = "columnfamily.outputtest";

    private static final String INPUT_FILE = "rows.txt";

    private static final int NUMBER_OF_ROWS = 3;
    private static final int NUMBER_OF_COLUMNS = 4;

    List<Integer> rowKeys;
    List<Integer> columnValues;

    private static FileSystem fs;

    static
    {
        try
        {
            fs = FileSystem.getLocal(new Configuration());
        }
        catch (IOException ioe)
        {
            throw new IOError(ioe);
        }
    }

    private Cassandra.Client thriftClient;

    @Before
    public void setup() throws Exception
    {
        setupCassandra();
        defineRows();
        createInputFile();
    }

    @After
    public void tearDown() throws Exception
    {
        deleteInputFile();
        deleteRows();
    }

    private void deleteRows() throws InvalidRequestException, TException
    {
        thriftClient.system_drop_column_family(COLUMN_FAMILY);
    }

    private void deleteInputFile() throws IOException
    {
        inputdir = new Path(INPUT_FOLDER);
        if (!fs.delete(inputdir, true))
            throw new IOException("Delete failed to remove " + inputdir.toString());
    }

    private void setupCassandra() throws TException, InvalidRequestException
    {
        /* Establish a thrift connection to the cassandra instance */
        TSocket socket = new TSocket(DatabaseDescriptor.getListenAddress().getHostName(), DatabaseDescriptor.getRpcPort());
        TTransport transport = new TFramedTransport(socket);
        TBinaryProtocol binaryProtocol = new TBinaryProtocol(transport, false, false);
        Cassandra.Client cassandraClient = new Cassandra.Client(binaryProtocol);
        transport.open();
{'fraction_non_alphanumeric': 0.08551406985656411, 'fraction_numerical': 0.00131391656629804, 'mean_word_length': 3.8048395581273016, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2771070', 'n_tokens_mistral': 2522, 'n_tokens_neox': 2498, 'n_words': 751}