<commit_before>package com.webimageloader.loader;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Iterator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import com.webimageloader.concurrent.ExecutorHelper;
import com.webimageloader.concurrent.ListenerFuture;
import com.webimageloader.util.DiskLruCache;
import com.webimageloader.util.Hasher;
import com.webimageloader.util.IOUtil;
import com.webimageloader.util.PriorityThreadFactory;
import com.webimageloader.util.DiskLruCache.Editor;
import com.webimageloader.util.DiskLruCache.Snapshot;

import android.graphics.Bitmap;
import android.os.Process;
import android.util.Log;

public class DiskLoader implements Loader, Closeable {
    private static final String TAG = "DiskLoader";

    private static final int APP_VERSION = 1;

    private static final int DEFAULT_BUFFER_SIZE = 8192;

    private static final Bitmap.CompressFormat COMPRESS_FORMAT = Bitmap.CompressFormat.JPEG;
    private static final int COMPRESS_QUALITY = 75;

    private static final int INPUT_IMAGE = 0;
    private static final int INPUT_METADATA = 1;
    private static final int VALUE_COUNT = 1;

    private ExecutorHelper executorHelper;

    private DiskLruCache cache;
    private Hasher hasher;

    public static DiskLoader open(File directory, long maxSize) throws IOException {
        return new DiskLoader(DiskLruCache.open(directory, APP_VERSION, VALUE_COUNT, maxSize));
    }

    private DiskLoader(DiskLruCache cache) {
        this.cache = cache;

        ExecutorService executor = Executors.newSingleThreadExecutor(new PriorityThreadFactory("Disk", Process.THREAD_PRIORITY_BACKGROUND));
        executorHelper = new ExecutorHelper(executor);

        hasher = new Hasher();
    }

    @Override
    public void close() {
        executorHelper.shutdown();

        IOUtil.closeQuietly(cache);
    }

    @Override
    public void load(final LoaderRequest request, final Iterator<Loader> chain, final Listener listener) {
        executorHelper.run(request, listener, new ListenerFuture.Task() {
            @Override
            public void run(Listener listener) throws Exception {
                String key = hashKeyForDisk(request);
                Snapshot snapshot = cache.get(key);
                if (snapshot != null) {
                    try {
                        Log.v(TAG, "Loaded " + request + " from disk");

                        InputStream is = snapshot.getInputStream(INPUT_IMAGE);
                        listener.onStreamLoaded(is);
                        is.close();

                    } finally {
                        snapshot.close();
                    }
                } else {
                    // We need to get the next loader
                    Loader next = chain.next();
                    next.load(request, chain, new NextListener(request, listener));
                }
            }
        });
    }

    @Override
    public void cancel(LoaderRequest request) {
        executorHelper.cancel(request);
    }

    private class NextListener implements Listener {
        private LoaderRequest request;
        private Listener listener;

        public NextListener(LoaderRequest request, Listener listener) {
            this.request = request;
            this.listener = listener;
        }

        @Override
        public void onStreamLoaded(InputStream is) {
            try {
                final String key = hashKeyForDisk(request);
                Editor editor = cache.edit(key);
                if (editor == null) {
                    throw new IOException("File is already being edited");
                }

                try {
                    OutputStream os = new BufferedOutputStream(editor.newOutputStream(INPUT_IMAGE));
                    copy(new BufferedInputStream(is), os);
                    os.close();
                    editor.commit();

                    // Read back the file we just saved
                    executorHelper.run(request, listener, new ReadTask(request));
                } catch (IOException e) {
                    editor.abort();
                    throw e;
                }
            } catch (IOException e) {
                listener.onError(e);
            }
        }

        @Override
        public void onBitmapLoaded(Bitmap b) {
            try {
                final String key = hashKeyForDisk(request);
                Editor editor = cache.edit(key);
                if (editor == null) {
                    throw new IOException("File is already being edited");
                }

                try {
                    OutputStream os = new BufferedOutputStream(editor.newOutputStream(INPUT_IMAGE));
                    b.compress(COMPRESS_FORMAT, COMPRESS_QUALITY, os);
                    os.close();
                    editor.commit();

                    // Read back the file we just saved
                    executorHelper.run(request, listener, new ReadTask(request));
                } catch (IOException e) {
                    editor.abort();
                    throw e;
                }
            } catch (IOException e) {
                listener.onError(e);
            }
        }

        @Override
        public void onError(Throwable t) {
            listener.onError(t);
        }
    }

    private class ReadTask implements ListenerFuture.Task {
        private LoaderRequest request;

        public ReadTask(LoaderRequest request) {<commit_msg>No need to read the file back when we already have it<commit_after>package com.webimageloader.loader;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Iterator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import com.webimageloader.concurrent.ExecutorHelper;
import com.webimageloader.concurrent.ListenerFuture;
import com.webimageloader.util.DiskLruCache;
import com.webimageloader.util.Hasher;
import com.webimageloader.util.IOUtil;
import com.webimageloader.util.PriorityThreadFactory;
import com.webimageloader.util.DiskLruCache.Editor;
import com.webimageloader.util.DiskLruCache.Snapshot;

import android.graphics.Bitmap;
import android.os.Process;
import android.util.Log;

public class DiskLoader implements Loader, Closeable {
    private static final String TAG = "DiskLoader";

    private static final int APP_VERSION = 1;

    private static final int DEFAULT_BUFFER_SIZE = 8192;

    private static final Bitmap.CompressFormat COMPRESS_FORMAT = Bitmap.CompressFormat.JPEG;
    private static final int COMPRESS_QUALITY = 75;

    private static final int INPUT_IMAGE = 0;
    private static final int INPUT_METADATA = 1;
    private static final int VALUE_COUNT = 1;

    private ExecutorHelper executorHelper;

    private DiskLruCache cache;
    private Hasher hasher;

    public static DiskLoader open(File directory, long maxSize) throws IOException {
        return new DiskLoader(DiskLruCache.open(directory, APP_VERSION, VALUE_COUNT, maxSize));
    }

    private DiskLoader(DiskLruCache cache) {
        this.cache = cache;

        ExecutorService executor = Executors.newSingleThreadExecutor(new PriorityThreadFactory("Disk", Process.THREAD_PRIORITY_BACKGROUND));
        executorHelper = new ExecutorHelper(executor);

        hasher = new Hasher();
    }

    @Override
    public void close() {
        executorHelper.shutdown();

        IOUtil.closeQuietly(cache);
    }

    @Override
    public void load(final LoaderRequest request, final Iterator<Loader> chain, final Listener listener) {
        executorHelper.run(request, listener, new ListenerFuture.Task() {
            @Override
            public void run(Listener listener) throws Exception {
                String key = hashKeyForDisk(request);
                Snapshot snapshot = cache.get(key);
                if (snapshot != null) {
                    try {
                        Log.v(TAG, "Loaded " + request + " from disk");

                        InputStream is = snapshot.getInputStream(INPUT_IMAGE);
                        listener.onStreamLoaded(is);
                        is.close();

                    } finally {
                        snapshot.close();
                    }
                } else {
                    // We need to get the next loader
                    Loader next = chain.next();
                    next.load(request, chain, new NextListener(request, listener));
                }
            }
        });
    }

    @Override
    public void cancel(LoaderRequest request) {
        executorHelper.cancel(request);
    }

    private class NextListener implements Listener {
        private LoaderRequest request;
        private Listener listener;

        public NextListener(LoaderRequest request, Listener listener) {
            this.request = request;
            this.listener = listener;
        }

        @Override
        public void onStreamLoaded(InputStream is) {
            try {
                final String key = hashKeyForDisk(request);
                Editor editor = cache.edit(key);
                if (editor == null) {
                    throw new IOException("File is already being edited");
                }

                try {
                    OutputStream os = new BufferedOutputStream(editor.newOutputStream(INPUT_IMAGE));
                    copy(new BufferedInputStream(is), os);
                    os.close();
                    editor.commit();

                    // Read back the file we just saved
                    executorHelper.run(request, listener, new ReadTask(request));
                } catch (IOException e) {
                    editor.abort();
                    throw e;
                }
            } catch (IOException e) {
                listener.onError(e);
            }
        }

        @Override
        public void onBitmapLoaded(Bitmap b) {
            try {
                final String key = hashKeyForDisk(request);
                Editor editor = cache.edit(key);
                if (editor == null) {
                    throw new IOException("File is already being edited");
                }

                try {
                    OutputStream os = new BufferedOutputStream(editor.newOutputStream(INPUT_IMAGE));
                    b.compress(COMPRESS_FORMAT, COMPRESS_QUALITY, os);
                    os.close();
                    editor.commit();

                    // Send the same bitmap to our listener
                    listener.onBitmapLoaded(b);
                } catch (IOException e) {
                    editor.abort();
                    throw e;
                }
            } catch (IOException e) {
                listener.onError(e);
            }
        }

        @Override
        public void onError(Throwable t) {
            listener.onError(t);
        }
    }

    private class ReadTask implements ListenerFuture.Task {
        private LoaderRequest request;

        public ReadTask(LoaderRequest request) {
{'fraction_non_alphanumeric': 0.07757255936675461, 'fraction_numerical': 0.001759014951627089, 'mean_word_length': 1.8882397764795529, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3434214', 'n_tokens_mistral': 2910, 'n_tokens_neox': 2707, 'n_words': 720}