<commit_before>/**
 * @file ARUTILS_Http.c
 * @brief libARUtils Http c file.
 * @date 26/12/2013
 * @author <EMAIL>
 **/

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>

#include <libARSAL/ARSAL_Sem.h>
#include <libARSAL/ARSAL_Print.h>
#include <curl/curl.h>

#include "libARUtils/ARUTILS_Error.h"
#include "libARUtils/ARUTILS_Http.h"
#include "libARUtils/ARUTILS_FileSystem.h"
#include "ARUTILS_Http.h"

#define ARUTILS_HTTP_TAG              "Http"

#ifdef DEBUG
#define ARUTILS_HTTP_CURL_VERBOSE         1
#endif

// Doc
//http://curl.haxx.se/libcurl/c/libcurl-easy.html

/*****************************************
 *
 *             Public implementation:
 *
 *****************************************/

ARUTILS_Http_Connection_t * ARUTILS_Http_Connection_New(ARSAL_Sem_t *cancelSem, const char *server, int port, eARUTILS_HTTPS_PROTOCOL security, const char *username, const char* password, eARUTILS_ERROR *error)
{
    ARUTILS_Http_Connection_t *newConnection = NULL;
    eARUTILS_ERROR result = ARUTILS_OK;

    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "%s, %d, %s", server ? server : "null", port, username ? username : "null");

    if (server == NULL)
    {
        result = ARUTILS_ERROR_BAD_PARAMETER;
    }

    newConnection = (ARUTILS_Http_Connection_t *)calloc(1, sizeof(ARUTILS_Http_Connection_t));

    if (newConnection == NULL)
    {
        result = ARUTILS_ERROR_ALLOC;
    }

    if (result == ARUTILS_OK)
    {
        newConnection->cancelSem = cancelSem;
    }

    if (result == ARUTILS_OK)
    {
        if (security == HTTPS_PROTOCOL_FALSE)
        {
            sprintf(newConnection->serverUrl, "http://%s:%d/", server, port);
        }
        else
        {
            sprintf(newConnection->serverUrl, "https://%s:%d/", server, port);
        }
    }

    if ((result == ARUTILS_OK) && (username != NULL))
    {
        strncpy(newConnection->username, username, ARUTILS_HTTP_MAX_USER_SIZE);
        newConnection->username[ARUTILS_HTTP_MAX_USER_SIZE - 1] = '\0';
    }

    if ((result == ARUTILS_OK) && (password != NULL))
    {
        strncpy(newConnection->password, password, ARUTILS_HTTP_MAX_USER_SIZE);
        newConnection->password[ARUTILS_HTTP_MAX_USER_SIZE - 1] = '\0';
    }

    if (result == ARUTILS_OK)
    {
        newConnection->curl = curl_easy_init();

        if (newConnection->curl == NULL)
        {
            result = ARUTILS_ERROR_CURL_ALLOC;
        }
    }

    if (result != ARUTILS_OK)
    {
        ARUTILS_Http_Connection_Delete(&newConnection);
    }

    *error = result;
    return newConnection;
}

void ARUTILS_Http_Connection_Delete(ARUTILS_Http_Connection_t **connectionPtrAddr)
{
    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "");

    if (connectionPtrAddr != NULL)
    {
        ARUTILS_Http_Connection_t *connection = *connectionPtrAddr;

        if (connection != NULL)
        {
            if (connection->curl != NULL)
            {
                curl_easy_cleanup(connection->curl);
            }

            ARUTILS_Http_FreeCallbackData(&connection->cbdata);

            free(connection);
            *connectionPtrAddr = NULL;
        }
    }
}

eARUTILS_ERROR ARUTILS_Http_Connection_Cancel(ARUTILS_Http_Connection_t *connection)
{
    eARUTILS_ERROR result = ARUTILS_OK;
    int resutlSys = 0;

    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "");

    if ((connection == NULL) || (connection->cancelSem == NULL))
    {
        result = ARUTILS_ERROR_BAD_PARAMETER;
    }

    if (result == ARUTILS_OK)
    {
        resutlSys = ARSAL_Sem_Post(connection->cancelSem);

        if (resutlSys != 0)
        {
            result = ARUTILS_ERROR_SYSTEM;
        }
    }

    return result;
}

eARUTILS_ERROR ARUTILS_Http_Request(ARUTILS_Http_Connection_t *connection, const char *endUrl, const char *urlParams, char **data, int *dataLen)
{
    char requestUrl[ARUTILS_HTTP_MAX_URL_SIZE];
    eARUTILS_ERROR result = ARUTILS_OK;
    CURLcode code = CURLE_OK;
    long httpCode = 0L;
    
    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "%s, %s", endUrl ? endUrl : "null", urlParams ? urlParams : "null");
    
    if ((connection == NULL) || (connection->curl == NULL) || (endUrl == NULL))
    {
        result =  ARUTILS_ERROR_BAD_PARAMETER;
    }
    
    if ((data == NULL) || (dataLen == NULL))
    {
        result =  ARUTILS_ERROR_BAD_PARAMETER;
    }
    
    if (result == ARUTILS_OK)
    {
        result = ARUTILS_Http_IsCanceled(connection);
    }
    
    if (result == ARUTILS_OK)
    {
        result = ARUTILS_Http_ResetOptions(connection);
    }
    
    if (result == ARUTILS_OK)
    {
        strncpy(requestUrl, connection->serverUrl, ARUTILS_HTTP_MAX_URL_SIZE);
        requestUrl[ARUTILS_HTTP_MAX_URL_SIZE - 1] = '\0';
        if (endUrl != NULL)
        {
            strncat(requestUrl, endUrl, ARUTILS_HTTP_MAX_URL_SIZE - strlen(requestUrl) - 1);
        }
        if (urlParams != NULL)
        {
            strncat(requestUrl, urlParams, ARUTILS_HTTP_MAX_URL_SIZE - strlen(requestUrl) - 1);
        }
        
        code = curl_easy_setopt(connection->curl, CURLOPT_URL, requestUrl);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }
    
    if (result == ARUTILS_OK)
    {
        code = curl_easy_setopt(connection->curl, CURLOPT_WRITEDATA, connection);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }
    
    if (result == ARUTILS_OK)
    {
        code = curl_easy_setopt(connection->curl, CURLOPT_WRITEFUNCTION, ARUTILS_Http_WriteDataCallback);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }
    
    //libcurl process
    if (result == ARUTILS_OK)
    {
        code = curl_easy_perform(connection->curl);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_Http_GetErrorFromCode(connection, code);
        }
    }
    
    if (result == ARUTILS_OK)
    {
        code = curl_easy_getinfo(connection->curl, CURLINFO_RESPONSE_CODE, &httpCode);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_GETINFO;
        }
    }
    
    //result checking
    if ((result == ARUTILS_OK) && (connection->cbdata.error != ARUTILS_OK))
    {
        result = connection->cbdata.error;
    }
    
    if (result == ARUTILS_OK)
    {
        // GET OK (200)
        if (httpCode == 200)
        {
            
        }
        else if (httpCode == 401)
        {
            result = ARUTILS_ERROR_HTTP_AUTHORIZATION_REQUIRED;
        }
        else if (httpCode == 403)
        {
            result = ARUTILS_ERROR_HTTP_ACCESS_DENIED;
        }
        else
        {
            result = ARUTILS_ERROR_HTTP_CODE;
        }
    }
    
    if (result == ARUTILS_OK)
    {
        *dataLen = connection->cbdata.dataSize;
        *data = malloc((*dataLen) + 1);
        memcpy(*data, (char*)connection->cbdata.data, *dataLen);
        (*data)[*dataLen] = '\0';
    }

    //cleanup
    if (connection != NULL)
    {
        ARUTILS_Http_FreeCallbackData(&connection->cbdata);
    }
    
    return result;
}

eARUTILS_ERROR ARUTILS_Http_Get(ARUTILS_Http_Connection_t *connection, const char *namePath, const char *dstFile, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    return ARUTILS_Http_Get_Internal(connection, namePath, dstFile, NULL, NULL, progressCallback, progressArg);
}

eARUTILS_ERROR ARUTILS_Http_Get_WithBuffer(ARUTILS_Http_Connection_t *connection, const char *namePath, uint8_t **data, uint32_t *dataLen, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    return ARUTILS_Http_Get_Internal(connection, namePath, NULL, data, dataLen, progressCallback, progressArg);
}

eARUTILS_ERROR ARUTILS_Http_Get_Internal(ARUTILS_Http_Connection_t *connection, const char *namePath, const char *dstFile, uint8_t **data, uint32_t *dataLen, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    char fileUrl[ARUTILS_HTTP_MAX_URL_SIZE];
    eARUTILS_ERROR result = ARUTILS_OK;
    CURLcode code = CURLE_OK;
    long httpCode = 0L;
    double remoteSize = 0.f;
    uint32_t localSize = 0;

    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "%s, %s", namePath ? namePath : "null", dstFile ? dstFile : "null");

    if ((connection == NULL) || (connection->curl == NULL) || (namePath == NULL))
    {
        result =  ARUTILS_ERROR_BAD_PARAMETER;
    }

    if (dstFile != NULL)
    {
        if ((data != NULL) || (dataLen != NULL))
        {
            result =  ARUTILS_ERROR_BAD_PARAMETER;
        }
    }
    else
    {
        if ((data == NULL) || (dataLen == NULL))
        {
            result =  ARUTILS_ERROR_BAD_PARAMETER;
        }
    }

    if (result == ARUTILS_OK)
    {
        result = ARUTILS_Http_IsCanceled(connection);
    }

    if (result == ARUTILS_OK)
    {
        result = ARUTILS_Http_ResetOptions(connection);
    }

    if (result == ARUTILS_OK)
    {
        strncpy(fileUrl, connection->serverUrl, ARUTILS_HTTP_MAX_URL_SIZE);
        fileUrl[ARUTILS_HTTP_MAX_URL_SIZE - 1] = '\0';
        strncat(fileUrl, namePath, ARUTILS_HTTP_MAX_URL_SIZE - strlen(fileUrl) - 1);

        code = curl_easy_setopt(connection->curl, CURLOPT_URL, fileUrl);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }

    if ((result == ARUTILS_OK) && (dstFile != NULL))
    {
        connection->cbdata.file = fopen(dstFile, "wb");

        if (connection->cbdata.file == NULL)
        {
            result = ARUTILS_ERROR_SYSTEM;
        }
    }

    if (result == ARUTILS_OK)
    {
        code = curl_easy_setopt(connection->curl, CURLOPT_WRITEDATA, connection);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }

    if (result == ARUTILS_OK)
    {
        code = curl_easy_setopt(connection->curl, CURLOPT_WRITEFUNCTION, ARUTILS_Http_WriteDataCallback);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }

    if (progressCallback != NULL)
    {
        if (result == ARUTILS_OK)
        {
            connection->cbdata.progressCallback = progressCallback;
            connection->cbdata.progressArg = progressArg;

            code = curl_easy_setopt(connection->curl, CURLOPT_PROGRESSDATA, connection);

            if (code != CURLE_OK)
            {
                result = ARUTILS_ERROR_CURL_SETOPT;
            }
        }

        if (result == ARUTILS_OK)
        {
            code = curl_easy_setopt(connection->curl, CURLOPT_PROGRESSFUNCTION, ARUTILS_Http_ProgressCallback);

            if (code != CURLE_OK)
            {
                result = ARUTILS_ERROR_CURL_SETOPT;
            }
        }

        if (result == ARUTILS_OK)
        {
            code = curl_easy_setopt(connection->curl, CURLOPT_NOPROGRESS, 0L);

            if (code != CURLE_OK)
            {
                result = ARUTILS_ERROR_CURL_SETOPT;
            }
        }
    }

    //libcurl process
    if (result == ARUTILS_OK)
    {
        code = curl_easy_perform(connection->curl);

        if (code != CURLE_OK)
        {
            result = ARUTILS_Http_GetErrorFromCode(connection, code);
        }
    }

    if (result == ARUTILS_OK)
    {
        code = curl_easy_getinfo(connection->curl, CURLINFO_RESPONSE_CODE, &httpCode);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_GETINFO;
        }
    }

    if (result == ARUTILS_OK)
    {
        code = curl_easy_getinfo(connection->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &remoteSize);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_GETINFO;
        }
    }

    //result checking
    if ((result == ARUTILS_OK) && (connection->cbdata.error != ARUTILS_OK))
    {
        result = connection->cbdata.error;
    }

    if (result == ARUTILS_OK)
    {
        // GET OK (200)
        if (httpCode == 200)
        {
            if (dstFile != NULL)
            {
                fflush(connection->cbdata.file);
            }
        }
        else if (httpCode == 401)
        {
            result = ARUTILS_ERROR_HTTP_AUTHORIZATION_REQUIRED;
        }
        else if (httpCode == 403)
        {
            result = ARUTILS_ERROR_HTTP_ACCESS_DENIED;
        }
        else
        {
            result = ARUTILS_ERROR_HTTP_CODE;
        }
    }

    if ((result == ARUTILS_OK) && (dstFile != NULL))
    {
        result = ARUTILS_FileSystem_GetFileSize(dstFile, &localSize);

        if (result == ARUTILS_OK)
        {
            if (localSize != (uint32_t)remoteSize)
            {
                result = ARUTILS_ERROR_HTTP_SIZE;
            }
        }
    }

    //cleanup
    if (connection != NULL)
    {
        ARUTILS_Http_FreeCallbackData(&connection->cbdata);
    }

    return result;
}

eARUTILS_ERROR ARUTILS_Http_Put(ARUTILS_Http_Connection_t *connection, const char *namePath, const char *srcFile, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    return ARUTILS_Http_Put_Internal(connection, namePath, srcFile, NULL, 0, progressCallback, progressArg);
}

eARUTILS_ERROR ARUTILS_Http_Put_Internal(ARUTILS_Http_Connection_t *connection, const char *namePath, const char *srcFile, uint8_t *data, uint32_t dataLen, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    struct curl_httppost *formItems = NULL;
    struct curl_httppost *lastFormItem = NULL;
    char fileUrl[ARUTILS_HTTP_MAX_URL_SIZE];
    eARUTILS_ERROR result = ARUTILS_OK;
    CURLFORMcode formCode = CURL_FORMADD_OK;
    CURLcode code = CURLE_OK;
    long httpCode = 0L;
    uint32_t localSize = 0;
    <commit_msg>[DEBUG](iOS,Android) Fix http get with buffer<commit_after>/**
 * @file ARUTILS_Http.c
 * @brief libARUtils Http c file.
 * @date 26/12/2013
 * @author <EMAIL>
 **/

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>

#include <libARSAL/ARSAL_Sem.h>
#include <libARSAL/ARSAL_Print.h>
#include <curl/curl.h>

#include "libARUtils/ARUTILS_Error.h"
#include "libARUtils/ARUTILS_Http.h"
#include "libARUtils/ARUTILS_FileSystem.h"
#include "ARUTILS_Http.h"

#define ARUTILS_HTTP_TAG              "Http"

#ifdef DEBUG
#define ARUTILS_HTTP_CURL_VERBOSE         1
#endif

// Doc
//http://curl.haxx.se/libcurl/c/libcurl-easy.html

/*****************************************
 *
 *             Public implementation:
 *
 *****************************************/

ARUTILS_Http_Connection_t * ARUTILS_Http_Connection_New(ARSAL_Sem_t *cancelSem, const char *server, int port, eARUTILS_HTTPS_PROTOCOL security, const char *username, const char* password, eARUTILS_ERROR *error)
{
    ARUTILS_Http_Connection_t *newConnection = NULL;
    eARUTILS_ERROR result = ARUTILS_OK;

    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "%s, %d, %s", server ? server : "null", port, username ? username : "null");

    if (server == NULL)
    {
        result = ARUTILS_ERROR_BAD_PARAMETER;
    }

    newConnection = (ARUTILS_Http_Connection_t *)calloc(1, sizeof(ARUTILS_Http_Connection_t));

    if (newConnection == NULL)
    {
        result = ARUTILS_ERROR_ALLOC;
    }

    if (result == ARUTILS_OK)
    {
        newConnection->cancelSem = cancelSem;
    }

    if (result == ARUTILS_OK)
    {
        if (security == HTTPS_PROTOCOL_FALSE)
        {
            sprintf(newConnection->serverUrl, "http://%s:%d/", server, port);
        }
        else
        {
            sprintf(newConnection->serverUrl, "https://%s:%d/", server, port);
        }
    }

    if ((result == ARUTILS_OK) && (username != NULL))
    {
        strncpy(newConnection->username, username, ARUTILS_HTTP_MAX_USER_SIZE);
        newConnection->username[ARUTILS_HTTP_MAX_USER_SIZE - 1] = '\0';
    }

    if ((result == ARUTILS_OK) && (password != NULL))
    {
        strncpy(newConnection->password, password, ARUTILS_HTTP_MAX_USER_SIZE);
        newConnection->password[ARUTILS_HTTP_MAX_USER_SIZE - 1] = '\0';
    }

    if (result == ARUTILS_OK)
    {
        newConnection->curl = curl_easy_init();

        if (newConnection->curl == NULL)
        {
            result = ARUTILS_ERROR_CURL_ALLOC;
        }
    }

    if (result != ARUTILS_OK)
    {
        ARUTILS_Http_Connection_Delete(&newConnection);
    }

    *error = result;
    return newConnection;
}

void ARUTILS_Http_Connection_Delete(ARUTILS_Http_Connection_t **connectionPtrAddr)
{
    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "");

    if (connectionPtrAddr != NULL)
    {
        ARUTILS_Http_Connection_t *connection = *connectionPtrAddr;

        if (connection != NULL)
        {
            if (connection->curl != NULL)
            {
                curl_easy_cleanup(connection->curl);
            }

            ARUTILS_Http_FreeCallbackData(&connection->cbdata);

            free(connection);
            *connectionPtrAddr = NULL;
        }
    }
}

eARUTILS_ERROR ARUTILS_Http_Connection_Cancel(ARUTILS_Http_Connection_t *connection)
{
    eARUTILS_ERROR result = ARUTILS_OK;
    int resutlSys = 0;

    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "");

    if ((connection == NULL) || (connection->cancelSem == NULL))
    {
        result = ARUTILS_ERROR_BAD_PARAMETER;
    }

    if (result == ARUTILS_OK)
    {
        resutlSys = ARSAL_Sem_Post(connection->cancelSem);

        if (resutlSys != 0)
        {
            result = ARUTILS_ERROR_SYSTEM;
        }
    }

    return result;
}

eARUTILS_ERROR ARUTILS_Http_Request(ARUTILS_Http_Connection_t *connection, const char *endUrl, const char *urlParams, char **data, int *dataLen)
{
    char requestUrl[ARUTILS_HTTP_MAX_URL_SIZE];
    eARUTILS_ERROR result = ARUTILS_OK;
    CURLcode code = CURLE_OK;
    long httpCode = 0L;
    
    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "%s, %s", endUrl ? endUrl : "null", urlParams ? urlParams : "null");
    
    if ((connection == NULL) || (connection->curl == NULL) || (endUrl == NULL))
    {
        result =  ARUTILS_ERROR_BAD_PARAMETER;
    }
    
    if ((data == NULL) || (dataLen == NULL))
    {
        result =  ARUTILS_ERROR_BAD_PARAMETER;
    }
    
    if (result == ARUTILS_OK)
    {
        result = ARUTILS_Http_IsCanceled(connection);
    }
    
    if (result == ARUTILS_OK)
    {
        result = ARUTILS_Http_ResetOptions(connection);
    }
    
    if (result == ARUTILS_OK)
    {
        strncpy(requestUrl, connection->serverUrl, ARUTILS_HTTP_MAX_URL_SIZE);
        requestUrl[ARUTILS_HTTP_MAX_URL_SIZE - 1] = '\0';
        if (endUrl != NULL)
        {
            strncat(requestUrl, endUrl, ARUTILS_HTTP_MAX_URL_SIZE - strlen(requestUrl) - 1);
        }
        if (urlParams != NULL)
        {
            strncat(requestUrl, urlParams, ARUTILS_HTTP_MAX_URL_SIZE - strlen(requestUrl) - 1);
        }
        
        code = curl_easy_setopt(connection->curl, CURLOPT_URL, requestUrl);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }
    
    if (result == ARUTILS_OK)
    {
        code = curl_easy_setopt(connection->curl, CURLOPT_WRITEDATA, connection);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }
    
    if (result == ARUTILS_OK)
    {
        code = curl_easy_setopt(connection->curl, CURLOPT_WRITEFUNCTION, ARUTILS_Http_WriteDataCallback);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }
    
    //libcurl process
    if (result == ARUTILS_OK)
    {
        code = curl_easy_perform(connection->curl);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_Http_GetErrorFromCode(connection, code);
        }
    }
    
    if (result == ARUTILS_OK)
    {
        code = curl_easy_getinfo(connection->curl, CURLINFO_RESPONSE_CODE, &httpCode);
        
        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_GETINFO;
        }
    }
    
    //result checking
    if ((result == ARUTILS_OK) && (connection->cbdata.error != ARUTILS_OK))
    {
        result = connection->cbdata.error;
    }
    
    if (result == ARUTILS_OK)
    {
        // GET OK (200)
        if (httpCode == 200)
        {
            
        }
        else if (httpCode == 401)
        {
            result = ARUTILS_ERROR_HTTP_AUTHORIZATION_REQUIRED;
        }
        else if (httpCode == 403)
        {
            result = ARUTILS_ERROR_HTTP_ACCESS_DENIED;
        }
        else
        {
            result = ARUTILS_ERROR_HTTP_CODE;
        }
    }
    
    if (result == ARUTILS_OK)
    {
        *dataLen = connection->cbdata.dataSize;
        *data = malloc((*dataLen) + 1);
        memcpy(*data, (char*)connection->cbdata.data, *dataLen);
        (*data)[*dataLen] = '\0';
    }

    //cleanup
    if (connection != NULL)
    {
        ARUTILS_Http_FreeCallbackData(&connection->cbdata);
    }
    
    return result;
}

eARUTILS_ERROR ARUTILS_Http_Get(ARUTILS_Http_Connection_t *connection, const char *namePath, const char *dstFile, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    return ARUTILS_Http_Get_Internal(connection, namePath, dstFile, NULL, NULL, progressCallback, progressArg);
}

eARUTILS_ERROR ARUTILS_Http_Get_WithBuffer(ARUTILS_Http_Connection_t *connection, const char *namePath, uint8_t **data, uint32_t *dataLen, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    return ARUTILS_Http_Get_Internal(connection, namePath, NULL, data, dataLen, progressCallback, progressArg);
}

eARUTILS_ERROR ARUTILS_Http_Get_Internal(ARUTILS_Http_Connection_t *connection, const char *namePath, const char *dstFile, uint8_t **data, uint32_t *dataLen, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    char fileUrl[ARUTILS_HTTP_MAX_URL_SIZE];
    eARUTILS_ERROR result = ARUTILS_OK;
    CURLcode code = CURLE_OK;
    long httpCode = 0L;
    double remoteSize = 0.f;
    uint32_t localSize = 0;

    ARSAL_PRINT(ARSAL_PRINT_DEBUG, ARUTILS_HTTP_TAG, "%s, %s", namePath ? namePath : "null", dstFile ? dstFile : "null");

    if ((connection == NULL) || (connection->curl == NULL) || (namePath == NULL))
    {
        result =  ARUTILS_ERROR_BAD_PARAMETER;
    }

    if (dstFile != NULL)
    {
        if ((data != NULL) || (dataLen != NULL))
        {
            result =  ARUTILS_ERROR_BAD_PARAMETER;
        }
    }
    else
    {
        if ((data == NULL) || (dataLen == NULL))
        {
            result =  ARUTILS_ERROR_BAD_PARAMETER;
        }
    }

    if (result == ARUTILS_OK)
    {
        result = ARUTILS_Http_IsCanceled(connection);
    }

    if (result == ARUTILS_OK)
    {
        result = ARUTILS_Http_ResetOptions(connection);
    }

    if (result == ARUTILS_OK)
    {
        strncpy(fileUrl, connection->serverUrl, ARUTILS_HTTP_MAX_URL_SIZE);
        fileUrl[ARUTILS_HTTP_MAX_URL_SIZE - 1] = '\0';
        strncat(fileUrl, namePath, ARUTILS_HTTP_MAX_URL_SIZE - strlen(fileUrl) - 1);

        code = curl_easy_setopt(connection->curl, CURLOPT_URL, fileUrl);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }

    if ((result == ARUTILS_OK) && (dstFile != NULL))
    {
        connection->cbdata.file = fopen(dstFile, "wb");

        if (connection->cbdata.file == NULL)
        {
            result = ARUTILS_ERROR_SYSTEM;
        }
    }

    if (result == ARUTILS_OK)
    {
        code = curl_easy_setopt(connection->curl, CURLOPT_WRITEDATA, connection);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }

    if (result == ARUTILS_OK)
    {
        code = curl_easy_setopt(connection->curl, CURLOPT_WRITEFUNCTION, ARUTILS_Http_WriteDataCallback);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_SETOPT;
        }
    }

    if (progressCallback != NULL)
    {
        if (result == ARUTILS_OK)
        {
            connection->cbdata.progressCallback = progressCallback;
            connection->cbdata.progressArg = progressArg;

            code = curl_easy_setopt(connection->curl, CURLOPT_PROGRESSDATA, connection);

            if (code != CURLE_OK)
            {
                result = ARUTILS_ERROR_CURL_SETOPT;
            }
        }

        if (result == ARUTILS_OK)
        {
            code = curl_easy_setopt(connection->curl, CURLOPT_PROGRESSFUNCTION, ARUTILS_Http_ProgressCallback);

            if (code != CURLE_OK)
            {
                result = ARUTILS_ERROR_CURL_SETOPT;
            }
        }

        if (result == ARUTILS_OK)
        {
            code = curl_easy_setopt(connection->curl, CURLOPT_NOPROGRESS, 0L);

            if (code != CURLE_OK)
            {
                result = ARUTILS_ERROR_CURL_SETOPT;
            }
        }
    }

    //libcurl process
    if (result == ARUTILS_OK)
    {
        code = curl_easy_perform(connection->curl);

        if (code != CURLE_OK)
        {
            result = ARUTILS_Http_GetErrorFromCode(connection, code);
        }
    }

    if (result == ARUTILS_OK)
    {
        code = curl_easy_getinfo(connection->curl, CURLINFO_RESPONSE_CODE, &httpCode);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_GETINFO;
        }
    }

    if (result == ARUTILS_OK)
    {
        code = curl_easy_getinfo(connection->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &remoteSize);

        if (code != CURLE_OK)
        {
            result = ARUTILS_ERROR_CURL_GETINFO;
        }
    }

    //result checking
    if ((result == ARUTILS_OK) && (connection->cbdata.error != ARUTILS_OK))
    {
        result = connection->cbdata.error;
    }

    if (result == ARUTILS_OK)
    {
        // GET OK (200)
        if (httpCode == 200)
        {
            if (dstFile != NULL)
            {
                fflush(connection->cbdata.file);
            }
        }
        else if (httpCode == 401)
        {
            result = ARUTILS_ERROR_HTTP_AUTHORIZATION_REQUIRED;
        }
        else if (httpCode == 403)
        {
            result = ARUTILS_ERROR_HTTP_ACCESS_DENIED;
        }
        else
        {
            result = ARUTILS_ERROR_HTTP_CODE;
        }
    }

    if ((result == ARUTILS_OK) && (dstFile != NULL))
    {
        result = ARUTILS_FileSystem_GetFileSize(dstFile, &localSize);

        if (result == ARUTILS_OK)
        {
            if (localSize != (uint32_t)remoteSize)
            {
                result = ARUTILS_ERROR_HTTP_SIZE;
            }
        }
    }    
    
    if ((result == ARUTILS_OK) && (data != NULL) && (dataLen != NULL))
    {
        if (result == ARUTILS_OK)
        {
            if (connection->cbdata.dataSize != (uint32_t)remoteSize)
            {
                result = ARUTILS_ERROR_FTP_SIZE;
            }
        }
        
        if (result == ARUTILS_OK)
        {
            *data = connection->cbdata.data;
            connection->cbdata.data = NULL;
            *dataLen = connection->cbdata.dataSize;
        }
    }

    //cleanup
    if (connection != NULL)
    {
        ARUTILS_Http_FreeCallbackData(&connection->cbdata);
    }

    return result;
}

eARUTILS_ERROR ARUTILS_Http_Put(ARUTILS_Http_Connection_t *connection, const char *namePath, const char *srcFile, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    return ARUTILS_Http_Put_Internal(connection, namePath, srcFile, NULL, 0, progressCallback, progressArg);
}

eARUTILS_ERROR ARUTILS_Http_Put_Internal(ARUTILS_Http_Connection_t *connection, const char *namePath, const char *srcFile, uint8_t *data, uint32_t dataLen, ARUTILS_Http_ProgressCallback_t progressCallback, void* progressArg)
{
    struct curl_httppost *formItems = NULL;
    struct curl_httppost *lastFormItem = NULL;
    char fileUrl[ARUTILS_HTTP_MAX_URL_SIZE];
    eARUTILS_ERROR result = ARUTILS_OK;
    CURLFORMcode formCode = CURL_FORMADD_OK;
    CURLcode code = CURLE_OK;
    long httpCode = 0L;
    uint32_t localSize = 0;
    
{'fraction_non_alphanumeric': 0.11120158606528358, 'fraction_numerical': 0.005168873468809743, 'mean_word_length': 2.319271445358402, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 115, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3911135', 'n_tokens_mistral': 10060, 'n_tokens_neox': 9665, 'n_words': 1617}