<commit_before>package mikesaelim.arxivoaiharvester;

import com.google.common.annotations.VisibleForTesting;
import mikesaelim.arxivoaiharvester.io.ArxivRequest;
import mikesaelim.arxivoaiharvester.io.ArxivResponse;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.xml.sax.SAXException;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import java.io.IOException;
import java.io.InputStream;

/**
 * TODO: big-ass description, javadoc on methods
 *
 * Created by donerkebab on 1/3/15.
 */
public class ArxivOAIHarvester {

    private final ArxivRequest arxivRequest;
    // TODO: other fields for flow control

    public ArxivOAIHarvester(ArxivRequest arxivRequest) {
        this.arxivRequest = arxivRequest;
    }

    public ArxivResponse getNextBatch() throws IOException, ClientProtocolException, Exception {
        // TODO: the Exception in the throws clause is temporary, only used until we handle response codes and flow control properly
        // TODO: persistent HttpClient instance?  Closeable objects?  HttpClient passed in, leading to unit tests?
        HttpClient httpClient = HttpClientBuilder.create().build();

        HttpGet httpRequest = new HttpGet(arxivRequest.getUri());
        httpRequest.addHeader("User-Agent", arxivRequest.getUserAgentHeader());
        httpRequest.addHeader("From", arxivRequest.getFromHeader());

        HttpResponse httpResponse = httpClient.execute(httpRequest);
        int httpStatusCode = httpResponse.getStatusLine().getStatusCode();

        // TODO: handle other response codes
        if (httpStatusCode != 200) {
            throw new Exception("Returned status code " + httpStatusCode + ": " +
                    httpResponse.getStatusLine().getReasonPhrase() + ": " +
                    EntityUtils.toString(httpResponse.getEntity()));
        }

        // TODO: implement resumption token storage and other flow control stuff

        return parseXMLStream(httpResponse.getEntity().getContent());
    }

    public boolean hasNextBatch() {
        // TODO: implement
        return false;
    }

    public ArxivRequest getArxivRequest() {
        return arxivRequest;
    }

    @VisibleForTesting ArxivResponse parseXMLStream(InputStream inputStream)
            throws ParserConfigurationException, SAXException, IOException {
        SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
        ArxivResponse.ArxivResponseBuilder responseBuilder = ArxivResponse.builder().arxivRequest(arxivRequest);

        parser.parse(inputStream, new XMLHandler(responseBuilder));

        return responseBuilder.build();
    }

}
<commit_msg>change to CloseableHttpClient/CloseableHttpResponse<commit_after>package mikesaelim.arxivoaiharvester;

import com.google.common.annotations.VisibleForTesting;
import mikesaelim.arxivoaiharvester.io.ArxivRequest;
import mikesaelim.arxivoaiharvester.io.ArxivResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.xml.sax.SAXException;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import java.io.IOException;
import java.io.InputStream;

/**
 * TODO: big-ass description, javadoc on methods
 *
 * Created by donerkebab on 1/3/15.
 */
public class ArxivOAIHarvester {

    private final ArxivRequest arxivRequest;
    // TODO: other fields for flow control

    public ArxivOAIHarvester(ArxivRequest arxivRequest) {
        this.arxivRequest = arxivRequest;
    }

    public ArxivResponse getNextBatch() throws IOException, ClientProtocolException, Exception {
        // TODO: the Exception in the throws clause is temporary, only used until we handle response codes and flow control properly
        // TODO: persistent HttpClient instance?  HttpClient passed in, leading to unit tests?
        ArxivResponse arxivResponse;

        try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
            HttpGet httpRequest = new HttpGet(arxivRequest.getUri());
            httpRequest.addHeader("User-Agent", arxivRequest.getUserAgentHeader());
            httpRequest.addHeader("From", arxivRequest.getFromHeader());

            try (CloseableHttpResponse httpResponse = httpClient.execute(httpRequest)) {
                int httpStatusCode = httpResponse.getStatusLine().getStatusCode();

                // TODO: handle other response codes
                if (httpStatusCode != 200) {
                    throw new Exception("Returned status code " + httpStatusCode + ": " +
                            httpResponse.getStatusLine().getReasonPhrase() + ": " +
                            EntityUtils.toString(httpResponse.getEntity()));
                }

                // TODO: implement resumption token storage and other flow control stuff

                arxivResponse = parseXMLStream(httpResponse.getEntity().getContent());
            }
        }

        return arxivResponse;
    }

    public boolean hasNextBatch() {
        // TODO: implement
        return false;
    }

    public ArxivRequest getArxivRequest() {
        return arxivRequest;
    }

    @VisibleForTesting ArxivResponse parseXMLStream(InputStream inputStream)
            throws ParserConfigurationException, SAXException, IOException {
        SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
        ArxivResponse.ArxivResponseBuilder responseBuilder = ArxivResponse.builder().arxivRequest(arxivRequest);

        parser.parse(inputStream, new XMLHandler(responseBuilder));

        return responseBuilder.build();
    }

}

{'fraction_non_alphanumeric': 0.08636955107351985, 'fraction_numerical': 0.002277163305139883, 'mean_word_length': 3.472, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4264157', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1568, 'n_words': 360}