<commit_before>#define PGDLLIMPORT "C"

#include <cstdlib>

#include "postgres.h"
#include "funcapi.h"

#include "access/extprotocol.h"
#include "catalog/pg_proc.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/memutils.h"
#include "fmgr.h"

#include "S3ExtWrapper.h"
#include "S3Common.h"
#include "S3Log.h"
#include "utils.h"

#include "gps3ext.h"
#include "gps3conf.h"

#include <pthread.h>
#include <openssl/err.h>

/* Do the module magic dance */

PG_MODULE_MAGIC;
PG_FUNCTION_INFO_V1(s3_export);
PG_FUNCTION_INFO_V1(s3_import);
PG_FUNCTION_INFO_V1(s3_validate_urls);

extern "C" {
Datum s3_export(PG_FUNCTION_ARGS);
Datum s3_import(PG_FUNCTION_ARGS);
Datum s3_validate_urls(PG_FUNCTION_ARGS);
}

#define MUTEX_TYPE pthread_mutex_t
#define MUTEX_SETUP(x) pthread_mutex_init(&(x), NULL)
#define MUTEX_CLEANUP(x) pthread_mutex_destroy(&(x))
#define MUTEX_LOCK(x) pthread_mutex_lock(&(x))
#define MUTEX_UNLOCK(x) pthread_mutex_unlock(&(x))
#define THREAD_ID pthread_self()

/* This array will store all of the mutexes available to OpenSSL. */
static MUTEX_TYPE *mutex_buf = NULL;

static void locking_function(int mode, int n, const char *file, int line) {
    if (mode & CRYPTO_LOCK)
        MUTEX_LOCK(mutex_buf[n]);
    else
        MUTEX_UNLOCK(mutex_buf[n]);
}

static unsigned long id_function(void) { return ((unsigned long)THREAD_ID); }

int thread_setup(void) {
    int i;

    mutex_buf =
        (pthread_mutex_t *)palloc(CRYPTO_num_locks() * sizeof(MUTEX_TYPE));
    if (!mutex_buf) return 0;
    for (i = 0; i < CRYPTO_num_locks(); i++) MUTEX_SETUP(mutex_buf[i]);
    CRYPTO_set_id_callback(id_function);
    CRYPTO_set_locking_callback(locking_function);
    return 1;
}

int thread_cleanup(void) {
    int i;

    if (!mutex_buf) return 0;
    CRYPTO_set_id_callback(NULL);
    CRYPTO_set_locking_callback(NULL);
    for (i = 0; i < CRYPTO_num_locks(); i++) MUTEX_CLEANUP(mutex_buf[i]);
    pfree(mutex_buf);
    mutex_buf = NULL;
    return 1;
}

/*
 * Import data into GPDB.
 */
Datum s3_import(PG_FUNCTION_ARGS) {
    S3ExtBase *myData;
    char *data;
    int data_len;
    size_t nread = 0;

    /* Must be called via the external table format manager */
    if (!CALLED_AS_EXTPROTOCOL(fcinfo))
        elog(ERROR,
             "extprotocol_import: not called by external protocol manager");

    /* Get our internal description of the protocol */
    myData = (S3ExtBase *)EXTPROTOCOL_GET_USER_CTX(fcinfo);

    if (EXTPROTOCOL_IS_LAST_CALL(fcinfo)) {
        if (myData) {
            thread_cleanup();
            if (!myData->Destroy()) {
                ereport(ERROR, (0, errmsg("Cleanup S3 extention failed")));
            }
            delete myData;
        }
        PG_RETURN_INT32(0);
    }

    if (myData == NULL) {
        /* first call. do any desired init */
        curl_global_init(CURL_GLOBAL_ALL);
        thread_setup();
        const char *p_name = "s3";
        char *url_with_options = EXTPROTOCOL_GET_URL(fcinfo);
        char *url = truncate_options(url_with_options);

        char *config_path = get_opt_s3(url_with_options, "config");
        if (!config_path) {
            // no config path in url, use default value
            // data_folder/gpseg0/s3/s3.conf
            config_path = strdup("s3/s3.conf");
        }

        bool result = InitConfig(config_path, "");
        if (!result) {
            ereport(ERROR,
                    (0, errmsg("Can't find config file %s", config_path)));
            free(config_path);
        } else {
            ClearConfig();
            free(config_path);
        }

        InitLog();

        if (s3ext_accessid == "") {
            ereport(ERROR, (0, errmsg("ERROR: access id is empty")));
        }<commit_msg>use pg alloc functions to avoid memory leaks when InitConfig() fails<commit_after>#define PGDLLIMPORT "C"

#include <cstdlib>

#include "postgres.h"
#include "funcapi.h"

#include "access/extprotocol.h"
#include "catalog/pg_proc.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/memutils.h"
#include "fmgr.h"

#include "S3ExtWrapper.h"
#include "S3Common.h"
#include "S3Log.h"
#include "utils.h"

#include "gps3ext.h"
#include "gps3conf.h"

#include <pthread.h>
#include <openssl/err.h>

/* Do the module magic dance */

PG_MODULE_MAGIC;
PG_FUNCTION_INFO_V1(s3_export);
PG_FUNCTION_INFO_V1(s3_import);
PG_FUNCTION_INFO_V1(s3_validate_urls);

extern "C" {
Datum s3_export(PG_FUNCTION_ARGS);
Datum s3_import(PG_FUNCTION_ARGS);
Datum s3_validate_urls(PG_FUNCTION_ARGS);
}

#define MUTEX_TYPE pthread_mutex_t
#define MUTEX_SETUP(x) pthread_mutex_init(&(x), NULL)
#define MUTEX_CLEANUP(x) pthread_mutex_destroy(&(x))
#define MUTEX_LOCK(x) pthread_mutex_lock(&(x))
#define MUTEX_UNLOCK(x) pthread_mutex_unlock(&(x))
#define THREAD_ID pthread_self()

/* This array will store all of the mutexes available to OpenSSL. */
static MUTEX_TYPE *mutex_buf = NULL;

static void locking_function(int mode, int n, const char *file, int line) {
    if (mode & CRYPTO_LOCK)
        MUTEX_LOCK(mutex_buf[n]);
    else
        MUTEX_UNLOCK(mutex_buf[n]);
}

static unsigned long id_function(void) { return ((unsigned long)THREAD_ID); }

int thread_setup(void) {
    int i;

    mutex_buf =
        (pthread_mutex_t *)palloc(CRYPTO_num_locks() * sizeof(MUTEX_TYPE));
    if (!mutex_buf) return 0;
    for (i = 0; i < CRYPTO_num_locks(); i++) MUTEX_SETUP(mutex_buf[i]);
    CRYPTO_set_id_callback(id_function);
    CRYPTO_set_locking_callback(locking_function);
    return 1;
}

int thread_cleanup(void) {
    int i;

    if (!mutex_buf) return 0;
    CRYPTO_set_id_callback(NULL);
    CRYPTO_set_locking_callback(NULL);
    for (i = 0; i < CRYPTO_num_locks(); i++) MUTEX_CLEANUP(mutex_buf[i]);
    pfree(mutex_buf);
    mutex_buf = NULL;
    return 1;
}

/*
 * Import data into GPDB.
 */
Datum s3_import(PG_FUNCTION_ARGS) {
    S3ExtBase *myData;
    char *data;
    int data_len;
    size_t nread = 0;

    /* Must be called via the external table format manager */
    if (!CALLED_AS_EXTPROTOCOL(fcinfo))
        elog(ERROR,
             "extprotocol_import: not called by external protocol manager");

    /* Get our internal description of the protocol */
    myData = (S3ExtBase *)EXTPROTOCOL_GET_USER_CTX(fcinfo);

    if (EXTPROTOCOL_IS_LAST_CALL(fcinfo)) {
        if (myData) {
            thread_cleanup();
            if (!myData->Destroy()) {
                ereport(ERROR, (0, errmsg("Cleanup S3 extention failed")));
            }
            delete myData;
        }
        PG_RETURN_INT32(0);
    }

    if (myData == NULL) {
        /* first call. do any desired init */
        curl_global_init(CURL_GLOBAL_ALL);
        thread_setup();
        const char *p_name = "s3";
        char *url_with_options = EXTPROTOCOL_GET_URL(fcinfo);
        char *url = truncate_options(url_with_options);

        char *config_path = get_opt_s3(url_with_options, "config");
        if (!config_path) {
            // no config path in url, use default value
            // data_folder/gpseg0/s3/s3.conf
            config_path = pstrdup("s3/s3.conf");
        }

        bool result = InitConfig(config_path, "");
        if (!result) {
            ereport(ERROR,
                    (0, errmsg("Can't find config file %s", config_path)));
            pfree(config_path);
        } else {
            ClearConfig();
            pfree(config_path);
        }

        InitLog();

        if (s3ext_accessid == "") {
            ereport(ERROR, (0, errmsg("ERROR: access id is empty")));
        }
{'fraction_non_alphanumeric': 0.12257893340408596, 'fraction_numerical': 0.010347572300344918, 'mean_word_length': 3.1264367816091956, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6151558', 'n_tokens_mistral': 2739, 'n_tokens_neox': 2684, 'n_words': 574}