<commit_before>package com.splunk.hunk.output;

import java.io.ByteArrayOutputStream;
import java.io.IOException;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.SequenceFile.Writer;
import org.apache.hadoop.io.Text;

public class StoresImages {

	public final Class<Text> keyClass = Text.class;
	public final Class<BytesWritable> valueClass = BytesWritable.class;
	private Path inputDir;
	private Path outputSeqFile;
	private FileSystem fs;

	public StoresImages(FileSystem fs, Path inputDir, Path outputSeqFile) {
		this.fs = fs;
		this.inputDir = inputDir;
		this.outputSeqFile = outputSeqFile;
	}

	public Path createSequenceFile() throws IOException {
		Writer writer = null;
		try {
			writer = createWriter();
			writeFiles(writer, fs.listStatus(inputDir));
		} finally {
			IOUtils.closeQuietly(writer);
		}
		return outputSeqFile;
	}

	private Writer createWriter() throws IOException {
		return SequenceFile.createWriter(fs, fs.getConf(), outputSeqFile,
				Text.class, valueClass, CompressionType.RECORD);
	}

	private void writeFiles(Writer writer, FileStatus[] listStatus)
			throws IOException {
		for (FileStatus f : listStatus)
			if (!f.isDir())
				appendFileContent(writer, f);
			else
				writeFiles(writer, fs.listStatus(f.getPath()));
	}

	private void appendFileContent(Writer writer, FileStatus f)
			throws IOException {
		Path p = f.getPath();
		ByteArrayOutputStream fileBytes = readFilesBytes(p);
		writer.append(new Text(p.toUri().getPath()), new BytesWritable(
				fileBytes.toByteArray()));
	}

	private ByteArrayOutputStream readFilesBytes(Path p) throws IOException {
		ByteArrayOutputStream fileBytes = null;
		FSDataInputStream open = null;
		try {
			fileBytes = new ByteArrayOutputStream();
			open = fs.open(p);
			IOUtils.copyLarge(open, fileBytes);
		} finally {
			IOUtils.closeQuietly(open);
			IOUtils.closeQuietly(fileBytes);
		}
		return fileBytes;
	}

}<commit_msg>Convenience method for creating image storer<commit_after>package com.splunk.hunk.output;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.SequenceFile.Writer;
import org.apache.hadoop.io.Text;

public class StoresImages {

	public final Class<Text> keyClass = Text.class;
	public final Class<BytesWritable> valueClass = BytesWritable.class;
	private Path inputDir;
	private Path outputSeqFile;
	private FileSystem fs;

	public StoresImages(FileSystem fs, Path inputDir, Path outputSeqFile) {
		this.fs = fs;
		this.inputDir = inputDir;
		this.outputSeqFile = outputSeqFile;
	}

	public Path createSequenceFile() throws IOException {
		Writer writer = null;
		try {
			writer = createWriter();
			writeFiles(writer, fs.listStatus(inputDir));
		} finally {
			IOUtils.closeQuietly(writer);
		}
		return outputSeqFile;
	}

	private Writer createWriter() throws IOException {
		return SequenceFile.createWriter(fs, fs.getConf(), outputSeqFile,
				Text.class, valueClass, CompressionType.RECORD);
	}

	private void writeFiles(Writer writer, FileStatus[] listStatus)
			throws IOException {
		for (FileStatus f : listStatus)
			if (!f.isDir())
				appendFileContent(writer, f);
			else
				writeFiles(writer, fs.listStatus(f.getPath()));
	}

	private void appendFileContent(Writer writer, FileStatus f)
			throws IOException {
		Path p = f.getPath();
		ByteArrayOutputStream fileBytes = readFilesBytes(p);
		writer.append(new Text(p.toUri().getPath()), new BytesWritable(
				fileBytes.toByteArray()));
	}

	private ByteArrayOutputStream readFilesBytes(Path p) throws IOException {
		ByteArrayOutputStream fileBytes = null;
		FSDataInputStream open = null;
		try {
			fileBytes = new ByteArrayOutputStream();
			open = fs.open(p);
			IOUtils.copyLarge(open, fileBytes);
		} finally {
			IOUtils.closeQuietly(open);
			IOUtils.closeQuietly(fileBytes);
		}
		return fileBytes;
	}

	public static StoresImages create(String uri, String dir, String output)
			throws IOException {
		return new StoresImages(FileSystem.get(URI.create(uri),
				new Configuration()), new Path(dir), new Path(output));
	}

}
{'fraction_non_alphanumeric': 0.10532915360501567, 'fraction_numerical': 0.0, 'mean_word_length': 4.923267326732673, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3097190', 'n_tokens_mistral': 1596, 'n_tokens_neox': 1475, 'n_words': 335}