<commit_before>package JpAws;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import java.io.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import system.FileSystem;

/**
 * This filesystem uses Amazon's S3 service, storing files in a bucket specified during construction.
 * @author <NAME>
 */
public class S3FileSystem extends FileSystem {
    private AmazonS3 s3 = new AmazonS3Client(PregelAuthenticator.get());

    public S3FileSystem(String jobDirectoryName) {
        super(jobDirectoryName);
    }

    @Override
    public BufferedReader getFileInputStream() throws FileNotFoundException {
        S3ObjectInputStream objectContent = s3.getObject(jobDirectoryName, "input").getObjectContent();
        return new BufferedReader(new InputStreamReader(objectContent));
    }

    @Override
    public BufferedWriter getFileOutputStream() {
        return new BufferedWriter(new OutputStreamWriter(inToOut("output")));
    }

    @Override
    public BufferedReader getWorkerInputFileInputStream(int WorkerNum) throws FileNotFoundException {
        String workerNumber = Integer.toString(WorkerNum);
        S3ObjectInputStream objectContent = s3.getObject(jobDirectoryName,"in/"+workerNumber).getObjectContent();
        return new BufferedReader(new InputStreamReader(objectContent));
    }

    @Override
    public BufferedWriter getWorkerInputFileOutputStream(int WorkerNum) {
        return new BufferedWriter(new OutputStreamWriter(inToOut("in/"+WorkerNum)));
    }

    @Override
    public BufferedReader getWorkerOutputFileInputStream(int WorkerNum) throws FileNotFoundException {
        String workerNumber = Integer.toString(WorkerNum);
        String bucketName = jobDirectoryName;
        S3ObjectInputStream objectContent = s3.getObject(bucketName,"out/"+workerNumber).getObjectContent();
        return new BufferedReader(new InputStreamReader(objectContent));
    }

    @Override
    public BufferedWriter getWorkerOutputFileOutputStream(int WorkerNum)  {
        return new BufferedWriter(new OutputStreamWriter(inToOut("/out/" + WorkerNum)));
    }
    
    private OutputStream inToOut(final String fileName) {
        final PipedInputStream in  = new PipedInputStream();
        PipedOutputStream out = null;
        try
        {
            out = new PipedOutputStream(in);
        } catch (IOException ex)
        {
            Logger.getLogger(S3FileSystem.class.getName()).log(Level.SEVERE, null, ex);
        }
        new Thread(new Runnable() {

                @Override
                public void run()
                {
                    s3.putObject(jobDirectoryName, fileName, in, null);
                }
            }).start();
        
        return out;
    }
}<commit_msg>Fixed a bug that broke S3.<commit_after>package JpAws;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import java.io.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import system.FileSystem;

/**
 * This filesystem uses Amazon's S3 service, storing files in a bucket specified during construction.
 * @author <NAME>
 */
public class S3FileSystem extends FileSystem {
    private AmazonS3 s3 = new AmazonS3Client(PregelAuthenticator.get());

    public S3FileSystem(String jobDirectoryName) {
        super(jobDirectoryName);
    }

    @Override
    public BufferedReader getFileInputStream() throws FileNotFoundException {
        S3ObjectInputStream objectContent = s3.getObject(jobDirectoryName, "input").getObjectContent();
        return new BufferedReader(new InputStreamReader(objectContent));
    }

    @Override
    public BufferedWriter getFileOutputStream() {
        return new BufferedWriter(new OutputStreamWriter(inToOut("output")));
    }

    @Override
    public BufferedReader getWorkerInputFileInputStream(int WorkerNum) throws FileNotFoundException {
        String workerNumber = Integer.toString(WorkerNum);
        S3ObjectInputStream objectContent = s3.getObject(jobDirectoryName,"in/"+workerNumber).getObjectContent();
        return new BufferedReader(new InputStreamReader(objectContent));
    }

    @Override
    public BufferedWriter getWorkerInputFileOutputStream(int WorkerNum) {
        return new BufferedWriter(new OutputStreamWriter(inToOut("in/"+WorkerNum)));
    }

    @Override
    public BufferedReader getWorkerOutputFileInputStream(int WorkerNum) throws FileNotFoundException {
        String workerNumber = Integer.toString(WorkerNum);
        String bucketName = jobDirectoryName;
        S3ObjectInputStream objectContent = s3.getObject(bucketName,"out/"+workerNumber).getObjectContent();
        return new BufferedReader(new InputStreamReader(objectContent));
    }

    @Override
    public BufferedWriter getWorkerOutputFileOutputStream(int WorkerNum)  {
        return new BufferedWriter(new OutputStreamWriter(inToOut("out/" + WorkerNum)));
    }
    
    private OutputStream inToOut(final String fileName) {
        final PipedInputStream in  = new PipedInputStream();
        PipedOutputStream out = null;
        try
        {
            out = new PipedOutputStream(in);
        } catch (IOException ex)
        {
            Logger.getLogger(S3FileSystem.class.getName()).log(Level.SEVERE, null, ex);
        }
        new Thread(new Runnable() {

                @Override
                public void run()
                {
                    s3.putObject(jobDirectoryName, fileName, in, null);
                }
            }).start();
        
        return out;
    }
}
{'fraction_non_alphanumeric': 0.08384458077709611, 'fraction_numerical': 0.006987048398091343, 'mean_word_length': 2.944220430107527, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4524885', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1501, 'n_words': 327}