<commit_before>package com.docussandra.javasdk.dao.impl.parent;

import com.docussandra.javasdk.Config;
import com.docussandra.javasdk.exceptions.RESTException;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClientBuilder;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author udeyoje
 */
public abstract class DaoParent
{

    private static final String L_DATABASES = "/databases";
    private static final String L_TABLES = "/tables";
    private static final String L_INDEXES = "/indexes";
    private static final String L_DOCUMENTS = "/documents";

    /**
     * Logger for this class.
     */
    private Logger logger = LoggerFactory.getLogger(this.getClass());

    /**
     * Config needed to make a call.
     */
    private Config config;

    /**
     * Our http client.
     */
    private HttpClient client;

    /**
     * Configurations for requests
     */
    private RequestConfig rc;

    private JSONParser parser = new JSONParser();

    public DaoParent(Config config)
    {
        this.config = config;
        client = HttpClientBuilder.create().build();
//        if (base64Encode) {
//            this.authToken = new String(Base64.encodeBase64(authToken.getBytes()));
//        } else {
//            this.authToken = authToken;
//        }
        rc = RequestConfig.custom().setConnectTimeout(60000).setSocketTimeout(3600000).setConnectionRequestTimeout(60000).build();
    }

    /**
     * Config needed to make a call.
     *
     * @return the config
     */
    public Config getConfig()
    {
        return config;
    }

    public String getBaseURL()
    {
        return config.getBaseUrl();
    }

    public String createFullURL(String route)
    {
        StringBuilder toReturn = new StringBuilder(route);
        if (toReturn.toString().startsWith("/"))
        {
            toReturn = new StringBuilder(toReturn.substring(1));//remove the first slash for consistancy, will be added back later
        }
        if (config.getFormat().equals(Config.Format.LONG)) //if it's long format; we have some more work to do
        {
            int slash = toReturn.indexOf("/");
            if (slash == -1)
            {
                slash = 0;
            }
            toReturn.insert(slash, L_DATABASES);
            slash = toReturn.indexOf("/", slash + L_DATABASES.length() + 1);
            if (slash != -1)
            {
                toReturn.insert(slash, L_TABLES);
            }
            //TODO: finish this
        }
        if (toReturn.toString().startsWith("/"))//ugly; fix!
        {
            toReturn = new StringBuilder(toReturn.substring(1));//remove the first slash for consistancy, will be added back later
        }
        toReturn.insert(0, getBaseURL());//prepend the baseurl
        return toReturn.toString();
    }

    /**
     * Does an http GET call.
     *
     * @param url to GET
     * @return JSONObject Representing the response. If the route returns no
     * body, the object will be empty.
     * @throws RESTException
     */
    protected JSONObject doGetCall(String url) throws RESTException
    {
        HttpGet request = new HttpGet(url);
        request.setConfig(rc);
        // add request header
        request.addHeader(HttpHeaders.CONTENT_TYPE, "application/json");
        //add auth if specified
        if (config.getSecToken() != null)
        {
            request.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + config.getSecToken());
        }
        String responseString = "";
        try
        {
            HttpResponse response = client.execute(request);
            int responseCode = response.getStatusLine().getStatusCode();
            if (responseCode < 200 || responseCode >= 300)
            {
                throw new RESTException("Error when doing a GET call agaist: " + url, response);
            }

            if (response.getEntity() != null)
            {
                responseString = IOUtils.toString(response.getEntity().getContent());
                return (JSONObject) parser.parse(responseString);
            } else
            {
                return new JSONObject();<commit_msg>added some logging<commit_after>package com.docussandra.javasdk.dao.impl.parent;

import com.docussandra.javasdk.Config;
import com.docussandra.javasdk.exceptions.RESTException;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClientBuilder;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author udeyoje
 */
public abstract class DaoParent
{

    private static final String L_DATABASES = "/databases";
    private static final String L_TABLES = "/tables";
    private static final String L_INDEXES = "/indexes";
    private static final String L_DOCUMENTS = "/documents";

    /**
     * Logger for this class.
     */
    private Logger logger = LoggerFactory.getLogger(this.getClass());

    /**
     * Config needed to make a call.
     */
    private Config config;

    /**
     * Our http client.
     */
    private HttpClient client;

    /**
     * Configurations for requests
     */
    private RequestConfig rc;

    private JSONParser parser = new JSONParser();

    public DaoParent(Config config)
    {
        this.config = config;
        client = HttpClientBuilder.create().build();
//        if (base64Encode) {
//            this.authToken = new String(Base64.encodeBase64(authToken.getBytes()));
//        } else {
//            this.authToken = authToken;
//        }
        rc = RequestConfig.custom().setConnectTimeout(60000).setSocketTimeout(3600000).setConnectionRequestTimeout(60000).build();
    }

    /**
     * Config needed to make a call.
     *
     * @return the config
     */
    public Config getConfig()
    {
        return config;
    }

    public String getBaseURL()
    {
        return config.getBaseUrl();
    }

    public String createFullURL(String route)
    {
        StringBuilder toReturn = new StringBuilder(route);
        if (toReturn.toString().startsWith("/"))
        {
            toReturn = new StringBuilder(toReturn.substring(1));//remove the first slash for consistancy, will be added back later
        }
        if (config.getFormat().equals(Config.Format.LONG)) //if it's long format; we have some more work to do
        {
            int slash = toReturn.indexOf("/");
            if (slash == -1)
            {
                slash = 0;
            }
            toReturn.insert(slash, L_DATABASES);
            slash = toReturn.indexOf("/", slash + L_DATABASES.length() + 1);
            if (slash != -1)
            {
                toReturn.insert(slash, L_TABLES);
            }
            //TODO: finish this
        }
        if (toReturn.toString().startsWith("/"))//ugly; fix!
        {
            toReturn = new StringBuilder(toReturn.substring(1));//remove the first slash for consistancy, will be added back later
        }
        toReturn.insert(0, getBaseURL());//prepend the baseurl
        return toReturn.toString();
    }

    /**
     * Does an http GET call.
     *
     * @param url to GET
     * @return JSONObject Representing the response. If the route returns no
     * body, the object will be empty.
     * @throws RESTException
     */
    protected JSONObject doGetCall(String url) throws RESTException
    {
        logger.debug("Attempting to GET: " + url);
        HttpGet request = new HttpGet(url);
        request.setConfig(rc);
        // add request header
        request.addHeader(HttpHeaders.CONTENT_TYPE, "application/json");
        //add auth if specified
        if (config.getSecToken() != null)
        {
            request.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + config.getSecToken());
        }
        String responseString = "";
        try
        {
            HttpResponse response = client.execute(request);
            int responseCode = response.getStatusLine().getStatusCode();
            if (responseCode < 200 || responseCode >= 300)
            {
                throw new RESTException("Error when doing a GET call agaist: " + url, response);
            }

            if (response.getEntity() != null)
            {
                responseString = IOUtils.toString(response.getEntity().getContent());
                logger.debug("Result from GET call: " + responseString);
                return (JSONObject) parser.parse(responseString);
            } else
            {
                return new JSONObject();
{'fraction_non_alphanumeric': 0.11033621317789112, 'fraction_numerical': 0.007910898303320495, 'mean_word_length': 2.265805574439157, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2893333', 'n_tokens_mistral': 2708, 'n_tokens_neox': 2617, 'n_words': 632}