<commit_before>package org.bkpathak.mapreduce.statuscount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by bijay on 11/27/14.
 */
public class LogProcessingTest {

  MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
  ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
  MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;

  @Before
  public void setUp() {
    IPCountMapper mapper = new IPCountMapper();
    LogStatusReducer reducer = new LogStatusReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
  }

  @Test
  public void testMapper() throws IOException {
    mapDriver.withInput(new LongWritable(), new Text("192.168.3.11 - - [2014-07-19T16:05:33Z] \"GET / HTTP/1.1\" " +
            "404 1081 \"-\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\""))
            .withOutput(new Text("404"), new IntWritable(1))
            .runTest();
  }

  @Test
  public void testReducer() throws IOException {
    List<IntWritable> values = new ArrayList<IntWritable>();
    values.add(new IntWritable(1));
    values.add(new IntWritable(1));
    values.add(new IntWritable(1));

    reduceDriver.withInput(new Text("404"), values)
            .withOutput(new Text("404"), new IntWritable(3))
            .runTest();
  }

  @Test
  public void testMapReduce() throws IOException {<commit_msg>updated code as classname is changed<commit_after>package org.bkpathak.mapreduce.statuscount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by bijay on 11/27/14.
 */
public class LogProcessingTest {

  MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
  ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
  MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;

  @Before
  public void setUp() {
    StatusCountMapper mapper = new StatusCountMapper();
    StatusCountReducer reducer = new StatusCountReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
  }

  @Test
  public void testMapper() throws IOException {
    mapDriver.withInput(new LongWritable(), new Text("192.168.3.11 - - [2014-07-19T16:05:33Z] \"GET / HTTP/1.1\" " +
            "404 1081 \"-\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\""))
            .withOutput(new Text("404"), new IntWritable(1))
            .runTest();
  }

  @Test
  public void testReducer() throws IOException {
    List<IntWritable> values = new ArrayList<IntWritable>();
    values.add(new IntWritable(1));
    values.add(new IntWritable(1));
    values.add(new IntWritable(1));

    reduceDriver.withInput(new Text("404"), values)
            .withOutput(new Text("404"), new IntWritable(3))
            .runTest();
  }

  @Test
  public void testMapReduce() throws IOException {
{'fraction_non_alphanumeric': 0.12626656274356976, 'fraction_numerical': 0.033775006495193556, 'mean_word_length': 4.092592592592593, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7661746', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1255, 'n_words': 263}