<commit_before>import java.io.*;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.text.*;
import java.util.Date;

import SevenZip.Compression.LZMA.Decoder;

public class FileList {
	private static FileList instance = null;
	private byte[] data;
	private int offset;

	private FileList() {
		try {
			ByteArrayOutputStream out = new ByteArrayOutputStream();
			RandomAccessFile in = new RandomAccessFile("files.lzma", "r");
			MappedByteBuffer buf = in.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, in.length());
			in.close();

			Decoder decoder = new Decoder();
			decoder.SetDecoderProperties(Unpack.readProps(buf));

			//from LzmaAlone.java
			int len = 0;
			for (int i = 0; i < 8; i++)
			{
				int v = buf.get() & 0xFF;
				len |= ((long)v) << (8 * i);
			}
			decoder.Code(buf, out, len);
			data = out.toByteArray();
			out.close();<commit_msg>fix jar execution (but it is very ugly now)<commit_after>import java.io.*;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.text.*;
import java.util.Date;

import SevenZip.Compression.LZMA.Decoder;

public class FileList {
	private static FileList instance = null;
	private byte[] data;
	private int offset;

	private FileList() {
		try {
			ByteArrayOutputStream out = new ByteArrayOutputStream();
			//TODO: less ugliness here
			File tmp = File.createTempFile("files", ".lzma");
			{
				InputStream tmpin = this.getClass().getClassLoader().getResourceAsStream("files.lzma");
				FileOutputStream tmpout = new FileOutputStream(tmp);
				byte[] buf = new byte[1024];
				int i = 0;
				while ((i = tmpin.read(buf)) != -1) {
					tmpout.write(buf, 0, i);
				}
			}
			FileInputStream in = new FileInputStream(tmp);
			MappedByteBuffer buf = in.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, in.getChannel().size());
			in.close();
			tmp.delete();

			Decoder decoder = new Decoder();
			decoder.SetDecoderProperties(Unpack.readProps(buf));

			//from LzmaAlone.java
			int len = 0;
			for (int i = 0; i < 8; i++)
			{
				int v = buf.get() & 0xFF;
				len |= ((long)v) << (8 * i);
			}
			decoder.Code(buf, out, len);
			data = out.toByteArray();
			out.close();
{'fraction_non_alphanumeric': 0.14558689717925385, 'fraction_numerical': 0.008644222020018199, 'mean_word_length': 3.649048625792812, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5723898', 'n_tokens_mistral': 856, 'n_tokens_neox': 724, 'n_words': 186}