<commit_before>package whelk.apix;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import javax.net.ssl.SSLSocketFactory;

/**
 * The purpose of this class is to provide a low level http connection that uses a TCP socket with sockopt SO_KEEPALIVE.
 * This is NOT to be confused with the HTTP keep-alive header, which is a completely different beast.
 *
 * SO_KEEPALIVE compels the OS to send periodic out-of-band (with respect to the TCP stream) keep-alive probes with the
 * ACK flag set. If an ACK is not received from the other end of the connection in a timely manner, the connection is
 * considered dead and an exception is raised.
 *
 * The specific use case for this is requests to APIX which have been known to loose connections without a clean
 * connection shut down (essentially leaving a client hanging for ever). Normal timeouts are not a good way to deal with
 * this, as requests can sometimes _legitimately_ take as much 16 hours during a Voyager-regen.
 */
public class LongTermHttpRequest
{
    private int m_responseCode;
    private String m_responseData;
    private HashMap<String, String> m_responseHeaders;

    public LongTermHttpRequest(String url, String verb, String contentType, String data)
            throws IOException
    {
        URL properUrl = new URL(url);
        int port = properUrl.getPort();
        if (port == -1)
            port = properUrl.getDefaultPort();
        if (port == -1)
            port = 80;

        try ( Socket socket = createSocket(properUrl.getProtocol(), properUrl.getHost(), port) )
        {
            socket.setKeepAlive(true); // Essentially the point of all this

            writeRequest( socket.getOutputStream(), properUrl.getHost(), properUrl.getPath(), verb, contentType, data );
            readResponse( socket.getInputStream() );
        }
    }

    public int getResponseCode()
    {
        return m_responseCode;
    }

    public String getResponseData()
    {
        return m_responseData;
    }

    public Map<String, String> getResponseHeaders()
    {
        return Collections.unmodifiableMap(m_responseHeaders);
    }

    private Socket createSocket(String protocol, String host, int port)
            throws IOException
    {
        if (protocol.equals("https"))
        {
            SSLSocketFactory ssf = (SSLSocketFactory) SSLSocketFactory.getDefault();
            return ssf.createSocket(host, port);
        }
        else
            return new Socket(host, port);
    }

    private void writeRequest(OutputStream outputStream, String host, String path, String verb, String contentType, String data)
            throws IOException
    {
        if (path.equals(""))
            path = "/";

        StringBuilder header = new StringBuilder();
        header.append( verb + " " + path + " HTTP/1.1\r\n" );
        header.append( "Host: " + host + "\r\n" );
        header.append( "Accept-Charset: utf-8\r\n" );
        header.append( "Connection: close\r\n" );

        byte[] dataBytes = null;
        if (data != null)
        {
            dataBytes = data.getBytes(Charset.forName("UTF-8"));
            header.append( "Content-Type: " + contentType + "\r\n" );
            header.append( "Content-Length: " + dataBytes.length + "\r\n" );
        }
        header.append( "\r\n" );

        outputStream.write(header.toString().getBytes(Charset.forName("UTF-8")));

        if (dataBytes != null)
            outputStream.write(dataBytes);

        outputStream.flush();
    }

    private void readResponse(InputStream inputStream)
            throws IOException
    {
        ByteArrayOutputStream completeResponse = new ByteArrayOutputStream();<commit_msg>Added BasicAuth support to LongTermHttpRequest.<commit_after>package whelk.apix;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import javax.net.ssl.SSLSocketFactory;

/**
 * The purpose of this class is to provide a low level http connection that uses a TCP socket with sockopt SO_KEEPALIVE.
 * This is NOT to be confused with the HTTP keep-alive header, which is a completely different beast.
 *
 * SO_KEEPALIVE compels the OS to send periodic out-of-band (with respect to the TCP stream) keep-alive probes with the
 * ACK flag set. If an ACK is not received from the other end of the connection in a timely manner, the connection is
 * considered dead and an exception is raised.
 *
 * The specific use case for this is requests to APIX which have been known to loose connections without a clean
 * connection shut down (essentially leaving a client hanging for ever). Normal timeouts are not a good way to deal with
 * this, as requests can sometimes _legitimately_ take as much 16 hours during a Voyager-regen.
 */
public class LongTermHttpRequest
{
    private int m_responseCode;
    private String m_responseData;
    private HashMap<String, String> m_responseHeaders;

    /**
     * contentType, data, basicAuthName and basicAuthPass may all be passed as null where they are not relevant.
     */
    public LongTermHttpRequest(String url, String verb, String contentType, String data,
                               String basicAuthName, String basicAuthPass)
            throws IOException
    {
        URL properUrl = new URL(url);
        int port = properUrl.getPort();
        if (port == -1)
            port = properUrl.getDefaultPort();
        if (port == -1)
            port = 80;

        try ( Socket socket = createSocket(properUrl.getProtocol(), properUrl.getHost(), port) )
        {
            socket.setKeepAlive(true); // Essentially the point of all this

            writeRequest( socket.getOutputStream(), properUrl.getHost(), properUrl.getPath(), verb, contentType,
                    data, basicAuthName, basicAuthPass );
            readResponse( socket.getInputStream() );
        }
    }

    public int getResponseCode()
    {
        return m_responseCode;
    }

    public String getResponseData()
    {
        return m_responseData;
    }

    public Map<String, String> getResponseHeaders()
    {
        return Collections.unmodifiableMap(m_responseHeaders);
    }

    private Socket createSocket(String protocol, String host, int port)
            throws IOException
    {
        if (protocol.equals("https"))
        {
            SSLSocketFactory ssf = (SSLSocketFactory) SSLSocketFactory.getDefault();
            return ssf.createSocket(host, port);
        }
        else
            return new Socket(host, port);
    }

    private void writeRequest(OutputStream outputStream, String host, String path, String verb, String contentType,
                              String data, String basicAuthName, String basicAuthPass)
            throws IOException
    {
        if (path.equals(""))
            path = "/";

        StringBuilder header = new StringBuilder();
        header.append( verb + " " + path + " HTTP/1.1\r\n" );
        header.append( "Host: " + host + "\r\n" );
        header.append( "Accept-Charset: utf-8\r\n" );
        header.append( "Connection: close\r\n" );

        if (basicAuthName != null && basicAuthPass != null)
        {
            Base64.Encoder encoder = Base64.getMimeEncoder(0, new byte[]{});
            String basicString = basicAuthName + ":" + basicAuthPass;
            String basicBase64 = new String(encoder.encode( basicString.getBytes("UTF-8")), Charset.forName("UTF-8"));
            header.append( "Authorization: Basic " + basicBase64 + "\r\n");
        }

        byte[] dataBytes = null;
        if (data != null)
        {
            dataBytes = data.getBytes(Charset.forName("UTF-8"));
            header.append( "Content-Type: " + contentType + "\r\n" );
            header.append( "Content-Length: " + dataBytes.length + "\r\n" );
        }
        header.append( "\r\n" );

        outputStream.write(header.toString().getBytes(Charset.forName("UTF-8")));

        if (dataBytes != null)
            outputStream.write(dataBytes);

        outputStream.flush();
    }

    private void readResponse(InputStream inputStream)
            throws IOException
    {
        ByteArrayOutputStream completeResponse = new ByteArrayOutputStream();
{'fraction_non_alphanumeric': 0.09633240482822655, 'fraction_numerical': 0.004062209842154132, 'mean_word_length': 2.616030214015946, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4891251', 'n_tokens_mistral': 2332, 'n_tokens_neox': 2288, 'n_words': 732}