<commit_before>package fr.liglab.mining.mapred;

import gnu.trove.map.TIntIntMap;
import gnu.trove.map.hash.TIntIntHashMap;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile.Reader;

/**
 * Tools for distributed cache I/O
 */
final class DistCache {
	static final String REBASINGMAP_DIRNAME = "rebasing";
	static final String PER_ITEM_BOUNDS_DIRNAME = "perItemBounds";
	
	/**
	 * Adds given path to conf's distributed cache
	 */
	static void copyToCache(Configuration conf, String path) throws IOException {
		FileSystem fs = FileSystem.get(conf);
		Path qualifiedPath = fs.makeQualified(new Path(path));
		FileStatus[] statuses = fs.listStatus(qualifiedPath);
		
		for (Path file : FileUtil.stat2Paths(statuses)) {
			String stringified = file.toString();
			int lastSlash = stringified.lastIndexOf('/');
			
			if (stringified.charAt(lastSlash+1) != '_') {
				DistributedCache.addCacheFile(file.toUri(), conf);
				System.out.println(file.toUri().toString());
			}
		}
	}
	
	static TIntIntMap readRebasing(Configuration conf) throws IOException {
		return readIntIntMap(conf, REBASINGMAP_DIRNAME);
	}
	
	static TIntIntMap readPerItemBounds(Configuration conf) throws IOException {
		return readIntIntMap(conf, PER_ITEM_BOUNDS_DIRNAME);
	}
	
	private static TIntIntMap readIntIntMap(Configuration conf, String token) throws IOException {
		TIntIntMap map = new TIntIntHashMap();
		
		FileSystem fs = FileSystem.getLocal(conf);
		
		for (Path path : DistributedCache.getLocalCacheFiles(conf)) {
			if (path.toString().contains(token)) {
				Reader reader = new Reader(fs, path, conf);
				IntWritable key = new IntWritable();
				IntWritable value = new IntWritable();
				
				while(reader.next(key, value)) {
					map.put(key.get(), value.get());
				}
				
				reader.close();<commit_msg>Looks good, finally !<commit_after>package fr.liglab.mining.mapred;

import gnu.trove.impl.Constants;
import gnu.trove.map.TIntIntMap;
import gnu.trove.map.hash.TIntIntHashMap;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile.Reader;

/**
 * Tools for distributed cache I/O
 */
final class DistCache {
	static final String REBASINGMAP_DIRNAME = "rebasing";
	static final String PER_ITEM_BOUNDS_DIRNAME = "perItemBounds";
	
	/**
	 * Adds given path to conf's distributed cache
	 */
	static void copyToCache(Configuration conf, String path) throws IOException {
		FileSystem fs = FileSystem.get(conf);
		Path qualifiedPath = fs.makeQualified(new Path(path));
		FileStatus[] statuses = fs.listStatus(qualifiedPath);
		
		for (Path file : FileUtil.stat2Paths(statuses)) {
			String stringified = file.toString();
			int lastSlash = stringified.lastIndexOf('/');
			
			if (stringified.charAt(lastSlash+1) != '_') {
				DistributedCache.addCacheFile(file.toUri(), conf);
				System.out.println(file.toUri().toString());
			}
		}
	}
	
	static TIntIntMap readRebasing(Configuration conf) throws IOException {
		return readIntIntMap(conf, REBASINGMAP_DIRNAME, conf.getInt(TopLCMoverHadoop.KEY_REBASING_MAX_ID, 10));
	}
	
	static TIntIntMap readPerItemBounds(Configuration conf) throws IOException {
		return readIntIntMap(conf, PER_ITEM_BOUNDS_DIRNAME, 1000);
	}
	
	private static TIntIntMap readIntIntMap(Configuration conf, String token, int size) throws IOException {
		TIntIntMap map = new TIntIntHashMap(size, Constants.DEFAULT_LOAD_FACTOR, -1, -1);
		FileSystem fs = FileSystem.getLocal(conf);
		
		for (Path path : DistributedCache.getLocalCacheFiles(conf)) {
			if (path.toString().contains(token)) {
				Reader reader = new Reader(fs, path, conf);
				IntWritable key = new IntWritable();
				IntWritable value = new IntWritable();
				
				while(reader.next(key, value)) {
					map.put(key.get(), value.get());
				}
				
				reader.close();
{'fraction_non_alphanumeric': 0.11513761467889909, 'fraction_numerical': 0.0027522935779816515, 'mean_word_length': 4.685788787483703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1687375', 'n_tokens_mistral': 1540, 'n_tokens_neox': 1365, 'n_words': 294}