<commit_before>/*
 * Copyright 2008-2009 LinkedIn, Inc
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package test.voldemort.contrib.batchindexer;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.List;

import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.ToolRunner;

import voldemort.contrib.batchindexer.ReadOnlyBatchIndexMapper;
import voldemort.contrib.batchindexer.ReadOnlyBatchIndexer;
import voldemort.serialization.DefaultSerializerFactory;
import voldemort.serialization.Serializer;
import voldemort.serialization.SerializerDefinition;
import voldemort.store.Store;
import voldemort.store.readonly.RandomAccessFileStore;
import voldemort.store.serialized.SerializingStore;
import voldemort.versioning.Versioned;

/**
 * Unit test to check Read-Only Batch Indexer <strong>in Local mode numReduce
 * will be only one hence we will see only one node files irrespective of
 * cluster details.</strong>
 * 
 * @author bbansal
 * 
 */
public class TestReadOnlyBatchIndexer extends TestCase {

    @SuppressWarnings("unchecked")
    public void testCSVFileBatchIndexer() throws Exception {

        ToolRunner.run(new Configuration(), new TextBatchIndexer(), null);

        // rename Files
        File dataDir = new File("contrib/batch-indexer/temp-output/text");
        new File(dataDir, "users.index_0").renameTo(new File(dataDir, "users.index"));
        new File(dataDir, "users.data_0").renameTo(new File(dataDir, "users.data"));
<commit_msg>TestReadOnlyBatchIndexer now deletes dataDir if already exists.<commit_after>/*
 * Copyright 2008-2009 LinkedIn, Inc
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package test.voldemort.contrib.batchindexer;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.List;

import junit.framework.TestCase;

import org.apache.commons.io.FileDeleteStrategy;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.ToolRunner;

import voldemort.contrib.batchindexer.ReadOnlyBatchIndexMapper;
import voldemort.contrib.batchindexer.ReadOnlyBatchIndexer;
import voldemort.serialization.DefaultSerializerFactory;
import voldemort.serialization.Serializer;
import voldemort.serialization.SerializerDefinition;
import voldemort.store.Store;
import voldemort.store.readonly.RandomAccessFileStore;
import voldemort.store.serialized.SerializingStore;
import voldemort.versioning.Versioned;

/**
 * Unit test to check Read-Only Batch Indexer <strong>in Local mode numReduce
 * will be only one hence we will see only one node files irrespective of
 * cluster details.</strong>
 * 
 * @author bbansal
 * 
 */
public class TestReadOnlyBatchIndexer extends TestCase {

    @SuppressWarnings("unchecked")
    public void testCSVFileBatchIndexer() throws Exception {

        // rename Files
        File dataDir = new File("contrib/batch-indexer/temp-output/text");
        if(dataDir.exists()) {
            FileDeleteStrategy.FORCE.delete(dataDir);
        }

        ToolRunner.run(new Configuration(), new TextBatchIndexer(), null);

        // rename Files
        new File(dataDir, "users.index_0").renameTo(new File(dataDir, "users.index"));
        new File(dataDir, "users.data_0").renameTo(new File(dataDir, "users.data"));

{'fraction_non_alphanumeric': 0.09433581939125177, 'fraction_numerical': 0.0056440233823825845, 'mean_word_length': 4.670857142857143, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3026114', 'n_tokens_mistral': 1441, 'n_tokens_neox': 1409, 'n_words': 396}