<commit_before>package com.splunk.hunk.input;

import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;

import javax.imageio.ImageIO;

import org.apache.commons.io.input.BoundedInputStream;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader;
import org.apache.log4j.Logger;

import com.splunk.hunk.input.ImageRecordReader.ImageEventProcessor;
import com.splunk.hunk.input.image.HsbBucketProcessor;
import com.splunk.mr.input.BaseSplunkRecordReader;
import com.splunk.mr.input.VixInputSplit;

public class SequenceImageRecordReader extends BaseSplunkRecordReader {

	private static final Logger gLogger = Logger
			.getLogger(SequenceImageRecordReader.class);
	private SequenceFileRecordReader<Text, BytesWritable> recordReader;
	private ImageEventProcessor imageProcessor;

	@Override
	public void vixInitialize(VixInputSplit split, TaskAttemptContext context)
			throws IOException, InterruptedException {

		recordReader = new SequenceFileRecordReader<Text, BytesWritable>();
		gLogger.info("Init: " + split.getPath());
		recordReader.initialize(split, context);
		imageProcessor = new HsbBucketProcessor();
	}

	@Override
	public Text getCurrentValue() throws IOException, InterruptedException {
		BufferedImage image = getImageFromValue(recordReader.getCurrentValue());
		String keyAsString = getKeyAsString();
		gLogger.info("Key: " + keyAsString);
		Map<String, Object> event;
		if (image != null) {
			event = imageProcessor.createEventFromImage(image);
		} else {
			event = new HashMap<String, Object>();
			event.put("value", "image was null");
		}
		event.put("image", keyAsString);
		return new Text(Utils.eventAsJson(event));
	}

	private String getKeyAsString() throws IOException, InterruptedException {
		return recordReader.getCurrentKey().toString();
	}

	private BufferedImage getImageFromValue(BytesWritable bytes)
			throws IOException, InterruptedException {
		gLogger.info("bytes len: " + bytes.getLength());
		try {
			return ImageIO.read(inputStreamFromBytesWritable(bytes));
		} catch (Exception e) {
			gLogger.info("Got exception when reading image: " + e.getMessage());
			return null;
		}
	}

	private BoundedInputStream inputStreamFromBytesWritable(BytesWritable bytes) {
		return new BoundedInputStream(
				new ByteArrayInputStream(bytes.getBytes()), bytes.getLength());
	}

	@Override
	public boolean nextKeyValue() throws IOException, InterruptedException {
		return recordReader.nextKeyValue();
	}

	@Override
	public String getName() {
		return "seq-image-reader";
	}

	@Override
	public Pattern getFilePattern() {
		return Pattern.compile("\\.seq$");
	}

	@Override
	public Text getCurrentKey() throws IOException, InterruptedException {
		throw new UnsupportedOperationException();
	}
}<commit_msg>SequenceImageRecordReader takes .map/data instead of .seq<commit_after>package com.splunk.hunk.input;

import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;

import javax.imageio.ImageIO;

import org.apache.commons.io.input.BoundedInputStream;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader;
import org.apache.log4j.Logger;

import com.splunk.hunk.input.ImageRecordReader.ImageEventProcessor;
import com.splunk.hunk.input.image.HsbBucketProcessor;
import com.splunk.mr.input.BaseSplunkRecordReader;
import com.splunk.mr.input.VixInputSplit;

public class SequenceImageRecordReader extends BaseSplunkRecordReader {

	private static final Logger gLogger = Logger
			.getLogger(SequenceImageRecordReader.class);
	private SequenceFileRecordReader<Text, BytesWritable> recordReader;
	private ImageEventProcessor imageProcessor;

	@Override
	public void vixInitialize(VixInputSplit split, TaskAttemptContext context)
			throws IOException, InterruptedException {
		recordReader = new SequenceFileRecordReader<Text, BytesWritable>();
		gLogger.info("Init: " + split.getPath());
		recordReader.initialize(split, context);
		imageProcessor = new HsbBucketProcessor();
	}

	@Override
	public Text getCurrentValue() throws IOException, InterruptedException {
		BufferedImage image = getImageFromValue(recordReader.getCurrentValue());
		String keyAsString = getKeyAsString();
		gLogger.info("Key: " + keyAsString);
		Map<String, Object> event;
		if (image != null) {
			event = imageProcessor.createEventFromImage(image);
		} else {
			event = new HashMap<String, Object>();
			event.put("value", "image was null");
		}
		event.put("image", keyAsString);
		return new Text(Utils.eventAsJson(event));
	}

	private String getKeyAsString() throws IOException, InterruptedException {
		return recordReader.getCurrentKey().toString();
	}

	private BufferedImage getImageFromValue(BytesWritable bytes)
			throws IOException, InterruptedException {
		gLogger.info("bytes len: " + bytes.getLength());
		try {
			return ImageIO.read(inputStreamFromBytesWritable(bytes));
		} catch (Exception e) {
			gLogger.info("Got exception when reading image: " + e.getMessage());
			return null;
		}
	}

	private BoundedInputStream inputStreamFromBytesWritable(BytesWritable bytes) {
		return new BoundedInputStream(
				new ByteArrayInputStream(bytes.getBytes()), bytes.getLength());
	}

	@Override
	public boolean nextKeyValue() throws IOException, InterruptedException {
		return recordReader.nextKeyValue();
	}

	@Override
	public String getName() {
		return "seq-image-reader";
	}

	@Override
	public Pattern getFilePattern() {
		return Pattern.compile("\\.map/data$");
	}

	@Override
	public Text getCurrentKey() throws IOException, InterruptedException {
		throw new UnsupportedOperationException();
	}
}
{'fraction_non_alphanumeric': 0.10401579986833442, 'fraction_numerical': 0.0003291639236339697, 'mean_word_length': 5.851183765501691, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3224014', 'n_tokens_mistral': 1866, 'n_tokens_neox': 1745, 'n_words': 377}