<commit_before>package eu.modernmt.context.lucene.storage;

import eu.modernmt.lang.LanguagePair;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;

import java.io.*;
import java.nio.file.Files;
import java.util.*;

import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;

/**
 * Created by davide on 22/09/16.
 */
public class CorporaIndex implements Closeable {

    public static CorporaIndex load(Options.AnalysisOptions analysisOptions, File indexFile, File bucketsFolder) throws IOException {
        BufferedReader reader = null;

        try {
            reader = new BufferedReader(new FileReader(indexFile));

            // Reading channels

            int length = Integer.parseInt(reader.readLine());
            HashMap<Short, Long> channels = new HashMap<>(length);

            for (int i = 0; i < length; i++) {
                String[] parts = reader.readLine().split(":");
                channels.put(Short.parseShort(parts[0]), Long.parseLong(parts[1]));
            }

            // Reading buckets

            length = Integer.parseInt(reader.readLine());
            ArrayList<CorpusBucket> buckets = new ArrayList<>(length);

            for (int i = 0; i < length; i++) {
                CorpusBucket bucket = CorpusBucket.deserialize(analysisOptions, bucketsFolder, reader);
                buckets.add(bucket);
            }

            // Creating result

            return new CorporaIndex(indexFile, analysisOptions, bucketsFolder, buckets, channels);
        } catch (RuntimeException e) {
            throw new IOException("Invalid index file at " + indexFile, e);
        } finally {
            IOUtils.closeQuietly(reader);
        }
    }

    private final File file;
    private final File swapFile;
    private final Options.AnalysisOptions analysisOptions;
    private final File bucketsFolder;
    private final HashMap<BucketKey, CorpusBucket> bucketByKey;
    private final HashMap<Long, HashSet<CorpusBucket>> bucketsByMemory;
    private final HashMap<Short, Long> channels;

    public CorporaIndex(File file, Options.AnalysisOptions analysisOptions, File bucketsFolder) {
        this(file, analysisOptions, bucketsFolder, Collections.emptyList(), new HashMap<>());
    }

    private CorporaIndex(File file, Options.AnalysisOptions analysisOptions, File bucketsFolder, Collection<CorpusBucket> buckets, HashMap<Short, Long> channels) {
        this.file = file;
        this.swapFile = new File(file.getParentFile(), "~" + file.getName());
        this.analysisOptions = analysisOptions;
        this.bucketsFolder = bucketsFolder;
        this.channels = channels;

        this.bucketByKey = new HashMap<>(buckets.size());
        this.bucketsByMemory = new HashMap<>(buckets.size());

        for (CorpusBucket bucket : buckets) {
            BucketKey key = BucketKey.forBucket(bucket);
            this.bucketByKey.put(key, bucket);
            this.bucketsByMemory.computeIfAbsent(bucket.getMemory(), k -> new HashSet<>()).add(bucket);
        }

    }

    public boolean registerData(short channel, long position) {
        Long existent = this.channels.get(channel);

        if (existent == null || position > existent) {
            this.channels.put(channel, position);
            return true;
        } else {
            return false;
        }
    }

    public CorpusBucket getBucket(LanguagePair direction, long memory) throws IOException {
        return getBucket(direction, memory, true);
    }

    public CorpusBucket getBucket(LanguagePair direction, long memory, boolean createIfAbsent) throws IOException {
        BucketKey key = new BucketKey(direction, memory);

        CorpusBucket bucket = bucketByKey.get(key);

        if (bucket == null) {
            if (!createIfAbsent)
                return null;

            bucket = new CorpusBucket(analysisOptions, bucketsFolder, direction, memory);

            this.bucketByKey.put(key, bucket);
            this.bucketsByMemory.computeIfAbsent(memory, k -> new HashSet<>()).add(bucket);
        }

        if (!bucket.isOpen())
            bucket.open();

        return bucket;
    }

    public Collection<CorpusBucket> getBucketsByMemory(long memory) {
        return this.bucketsByMemory.get(memory);
    }

    public void remove(CorpusBucket bucket) {
        Long memory = bucket.getMemory();
        BucketKey key = BucketKey.forBucket(bucket);

        bucketByKey.remove(key);
        HashSet<CorpusBucket> buckets = bucketsByMemory.get(memory);<commit_msg>Fixed NullPointerException on double-delete same memory<commit_after>package eu.modernmt.context.lucene.storage;

import eu.modernmt.lang.LanguagePair;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;

import java.io.*;
import java.nio.file.Files;
import java.util.*;

import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;

/**
 * Created by davide on 22/09/16.
 */
public class CorporaIndex implements Closeable {

    public static CorporaIndex load(Options.AnalysisOptions analysisOptions, File indexFile, File bucketsFolder) throws IOException {
        BufferedReader reader = null;

        try {
            reader = new BufferedReader(new FileReader(indexFile));

            // Reading channels

            int length = Integer.parseInt(reader.readLine());
            HashMap<Short, Long> channels = new HashMap<>(length);

            for (int i = 0; i < length; i++) {
                String[] parts = reader.readLine().split(":");
                channels.put(Short.parseShort(parts[0]), Long.parseLong(parts[1]));
            }

            // Reading buckets

            length = Integer.parseInt(reader.readLine());
            ArrayList<CorpusBucket> buckets = new ArrayList<>(length);

            for (int i = 0; i < length; i++) {
                CorpusBucket bucket = CorpusBucket.deserialize(analysisOptions, bucketsFolder, reader);
                buckets.add(bucket);
            }

            // Creating result

            return new CorporaIndex(indexFile, analysisOptions, bucketsFolder, buckets, channels);
        } catch (RuntimeException e) {
            throw new IOException("Invalid index file at " + indexFile, e);
        } finally {
            IOUtils.closeQuietly(reader);
        }
    }

    private final File file;
    private final File swapFile;
    private final Options.AnalysisOptions analysisOptions;
    private final File bucketsFolder;
    private final HashMap<BucketKey, CorpusBucket> bucketByKey;
    private final HashMap<Long, HashSet<CorpusBucket>> bucketsByMemory;
    private final HashMap<Short, Long> channels;

    public CorporaIndex(File file, Options.AnalysisOptions analysisOptions, File bucketsFolder) {
        this(file, analysisOptions, bucketsFolder, Collections.emptyList(), new HashMap<>());
    }

    private CorporaIndex(File file, Options.AnalysisOptions analysisOptions, File bucketsFolder, Collection<CorpusBucket> buckets, HashMap<Short, Long> channels) {
        this.file = file;
        this.swapFile = new File(file.getParentFile(), "~" + file.getName());
        this.analysisOptions = analysisOptions;
        this.bucketsFolder = bucketsFolder;
        this.channels = channels;

        this.bucketByKey = new HashMap<>(buckets.size());
        this.bucketsByMemory = new HashMap<>(buckets.size());

        for (CorpusBucket bucket : buckets) {
            BucketKey key = BucketKey.forBucket(bucket);
            this.bucketByKey.put(key, bucket);
            this.bucketsByMemory.computeIfAbsent(bucket.getMemory(), k -> new HashSet<>()).add(bucket);
        }

    }

    public boolean registerData(short channel, long position) {
        Long existent = this.channels.get(channel);

        if (existent == null || position > existent) {
            this.channels.put(channel, position);
            return true;
        } else {
            return false;
        }
    }

    public CorpusBucket getBucket(LanguagePair direction, long memory) throws IOException {
        return getBucket(direction, memory, true);
    }

    public CorpusBucket getBucket(LanguagePair direction, long memory, boolean createIfAbsent) throws IOException {
        BucketKey key = new BucketKey(direction, memory);

        CorpusBucket bucket = bucketByKey.get(key);

        if (bucket == null) {
            if (!createIfAbsent)
                return null;

            bucket = new CorpusBucket(analysisOptions, bucketsFolder, direction, memory);

            this.bucketByKey.put(key, bucket);
            this.bucketsByMemory.computeIfAbsent(memory, k -> new HashSet<>()).add(bucket);
        }

        if (!bucket.isOpen())
            bucket.open();

        return bucket;
    }

    public Collection<CorpusBucket> getBucketsByMemory(long memory) {
        Collection<CorpusBucket> result = this.bucketsByMemory.get(memory);
        return result == null ? Collections.emptyList() : result;
    }

    public void remove(CorpusBucket bucket) {
        Long memory = bucket.getMemory();
        BucketKey key = BucketKey.forBucket(bucket);

        bucketByKey.remove(key);
        HashSet<CorpusBucket> buckets = bucketsByMemory.get(memory);
{'fraction_non_alphanumeric': 0.10176754151044456, 'fraction_numerical': 0.0021424745581146223, 'mean_word_length': 2.7493975903614456, 'pattern_counts': {'":': 2, '<': 42, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2658806', 'n_tokens_mistral': 2518, 'n_tokens_neox': 2395, 'n_words': 602}