<commit_before>#include "condor_common.h"
#include "condor_classad.h"
#include "../condor_utils/file_transfer_stats.h"
#include "../condor_utils/condor_url.h"
#include "multifile_curl_plugin.h"
#include "utc_time.h"
#include <algorithm>
#include <exception>
#include <sstream>
#include <iostream>
#include <chrono>
#include <thread>
#include <fstream>
#include <cstdio>
#include <stdexcept>
#include <rapidjson/document.h>

#define MAX_RETRY_ATTEMPTS 20
int max_retry_attempts = MAX_RETRY_ATTEMPTS;

// Setup a transfer progress callback. We'll use this to manually timeout 
// any transfers that are not making forward progress.

struct xferProgress {
    double lastRunTime;
    CURL *curl;
};
struct xferProgress myProgress;

static int xferInfo(void *p, double /* dltotal */, double dlnow, double /* ultotal */, double ulnow)
{
	/* Note : we cannot rely on dltotal or ultotal being correct, since
	they will only be non-zero if the server responded with the optional
	Content-Length header.  */

    struct xferProgress *progress = (struct xferProgress *)p;
    CURL *curl = progress->curl;
    double curTime = 0;
    static double prevTime = 0; //Previous total time
    static double dlprev   = 0; //Previous dlnow
    static double ulprev   = 0; //Previous ulnow
    /*Get download and upload differences from previous numbers to determine
    if we should timeout since dlnow and ulnow are totals. This is to prevent
    a stall out in file tranfser after a large change.*/
    static double diffTime = 0;
    static double diff_dl  = 0;
    static double diff_ul  = 0;
    diff_dl += dlnow - dlprev;
    diff_ul += ulnow - ulprev;

    dlprev = dlnow;
    ulprev = ulnow;

    curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &curTime);
    diffTime += curTime - prevTime;
    prevTime = curTime;

    // After 30 seconds, check if we're making forward progress (> 1 byte/s)
    if (curTime > 30) {
        if (diffTime > 30) {
            // If this is a download and not making progress, abort
            if (dlnow > 0 && diffTime > diff_dl) return 1;

            // If this is an upload and not making progress, abort
            if (ulnow > 0 && diffTime > diff_ul) return 1;

            //Reset diffs for next check
            diffTime = 0;
            diff_dl  = 0;
            diff_ul  = 0;
        }
        // If not a single byte has been transferred either direction after 30 seconds, abort
        if (dlnow <= 0 && ulnow <= 0) return 1;
    }

    // All good. Return success.
    return 0;
}


namespace {

bool
ShouldRetryTransfer(int rval) {
    switch (rval) {
        case CURLE_COULDNT_CONNECT:
        case CURLE_PARTIAL_FILE:
        case CURLE_READ_ERROR:
        case CURLE_WRITE_ERROR:
        case CURLE_OPERATION_TIMEDOUT:
        case CURLE_SEND_ERROR:
        case CURLE_RECV_ERROR:
        case CURLE_GOT_NOTHING:
            return true;
        default:
            return false;
    };
}

extern "C"
size_t
CurlReadCallback(char *buffer, size_t size, size_t nitems, void *userdata) {
    if (userdata == nullptr) {
        return size*nitems;
    }
    return fread(buffer, size, nitems, static_cast<FILE*>(userdata));
}


extern "C"
size_t
CurlWriteCallback(char *buffer, size_t size, size_t nitems, void *userdata) {
    if (userdata == nullptr) {
        return size*nitems;
    }
    return fwrite(buffer, size, nitems, static_cast<FILE*>(userdata));
}

void
GetToken(const std::string & cred_name, std::string & token) {
	if (cred_name.empty()) {
		return;
	}
	const char *creddir = getenv("_CONDOR_CREDS");
	if (!creddir) {
		std::stringstream ss; ss << "Credential for " << cred_name << " requested by $_CONDOR_CREDS not set";
		throw std::runtime_error(ss.str());
	}

	// Cred name (via URL scheme) come in as <provider>[.<handle>]
	// but tokens are stored on disk as <provider>[_<handle>].use
	std::string cred_basename = cred_name;
	std::replace(cred_basename.begin(), cred_basename.end(), '.', '_');

	std::string cred_path = std::string(creddir) + DIR_DELIM_STRING + cred_basename + ".use";
	int fd = open(cred_path.c_str(), O_RDONLY);
	if (-1 == fd) {
		fprintf( stderr, "Error: Unable to open credential file %s: %s (errno=%d)", cred_path.c_str(),
			strerror(errno), errno);
		std::stringstream ss; ss << "Unable to open credential file " << cred_path << ": " << strerror(errno) << " (errno=" << errno << ")";
		throw std::runtime_error(ss.str());
	}
	close(fd);
	std::ifstream istr(cred_path, std::ios::binary);
	if (!istr.is_open()) {
		throw std::runtime_error("Failed to reopen credential file");
	}
	for (std::string line; std::getline(istr, line); ) {
		auto iter = line.begin();
		while (isspace(*iter)) {iter++;}
		if (*iter == '#') continue;
		rapidjson::Document doc;<commit_msg>HTCONDOR-1447: Multifile curl plugin scitoken parsing bug<commit_after>#include "condor_common.h"
#include "condor_classad.h"
#include "../condor_utils/file_transfer_stats.h"
#include "../condor_utils/condor_url.h"
#include "multifile_curl_plugin.h"
#include "utc_time.h"
#include <algorithm>
#include <exception>
#include <sstream>
#include <iostream>
#include <chrono>
#include <thread>
#include <fstream>
#include <cstdio>
#include <stdexcept>
#include <rapidjson/document.h>

#define MAX_RETRY_ATTEMPTS 20
int max_retry_attempts = MAX_RETRY_ATTEMPTS;

// Setup a transfer progress callback. We'll use this to manually timeout 
// any transfers that are not making forward progress.

struct xferProgress {
    double lastRunTime;
    CURL *curl;
};
struct xferProgress myProgress;

static int xferInfo(void *p, double /* dltotal */, double dlnow, double /* ultotal */, double ulnow)
{
	/* Note : we cannot rely on dltotal or ultotal being correct, since
	they will only be non-zero if the server responded with the optional
	Content-Length header.  */

    struct xferProgress *progress = (struct xferProgress *)p;
    CURL *curl = progress->curl;
    double curTime = 0;
    static double prevTime = 0; //Previous total time
    static double dlprev   = 0; //Previous dlnow
    static double ulprev   = 0; //Previous ulnow
    /*Get download and upload differences from previous numbers to determine
    if we should timeout since dlnow and ulnow are totals. This is to prevent
    a stall out in file tranfser after a large change.*/
    static double diffTime = 0;
    static double diff_dl  = 0;
    static double diff_ul  = 0;
    diff_dl += dlnow - dlprev;
    diff_ul += ulnow - ulprev;

    dlprev = dlnow;
    ulprev = ulnow;

    curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &curTime);
    diffTime += curTime - prevTime;
    prevTime = curTime;

    // After 30 seconds, check if we're making forward progress (> 1 byte/s)
    if (curTime > 30) {
        if (diffTime > 30) {
            // If this is a download and not making progress, abort
            if (dlnow > 0 && diffTime > diff_dl) return 1;

            // If this is an upload and not making progress, abort
            if (ulnow > 0 && diffTime > diff_ul) return 1;

            //Reset diffs for next check
            diffTime = 0;
            diff_dl  = 0;
            diff_ul  = 0;
        }
        // If not a single byte has been transferred either direction after 30 seconds, abort
        if (dlnow <= 0 && ulnow <= 0) return 1;
    }

    // All good. Return success.
    return 0;
}


namespace {

bool
ShouldRetryTransfer(int rval) {
    switch (rval) {
        case CURLE_COULDNT_CONNECT:
        case CURLE_PARTIAL_FILE:
        case CURLE_READ_ERROR:
        case CURLE_WRITE_ERROR:
        case CURLE_OPERATION_TIMEDOUT:
        case CURLE_SEND_ERROR:
        case CURLE_RECV_ERROR:
        case CURLE_GOT_NOTHING:
            return true;
        default:
            return false;
    };
}

extern "C"
size_t
CurlReadCallback(char *buffer, size_t size, size_t nitems, void *userdata) {
    if (userdata == nullptr) {
        return size*nitems;
    }
    return fread(buffer, size, nitems, static_cast<FILE*>(userdata));
}


extern "C"
size_t
CurlWriteCallback(char *buffer, size_t size, size_t nitems, void *userdata) {
    if (userdata == nullptr) {
        return size*nitems;
    }
    return fwrite(buffer, size, nitems, static_cast<FILE*>(userdata));
}

void
GetToken(const std::string & cred_name, std::string & token) {
	if (cred_name.empty()) {
		return;
	}
	const char *creddir = getenv("_CONDOR_CREDS");
	if (!creddir) {
		std::stringstream ss; ss << "Credential for " << cred_name << " requested by $_CONDOR_CREDS not set";
		throw std::runtime_error(ss.str());
	}

	// Cred name (via URL scheme) come in as <provider>[.<handle>]
	// but tokens are stored on disk as <provider>[_<handle>].use
	std::string cred_basename = cred_name;
	std::replace(cred_basename.begin(), cred_basename.end(), '.', '_');

	std::string cred_path = std::string(creddir) + DIR_DELIM_STRING + cred_basename + ".use";
	int fd = open(cred_path.c_str(), O_RDONLY);
	if (-1 == fd) {
		fprintf( stderr, "Error: Unable to open credential file %s: %s (errno=%d)", cred_path.c_str(),
			strerror(errno), errno);
		std::stringstream ss; ss << "Unable to open credential file " << cred_path << ": " << strerror(errno) << " (errno=" << errno << ")";
		throw std::runtime_error(ss.str());
	}
	close(fd);
	std::ifstream istr(cred_path, std::ios::binary);
	if (!istr.is_open()) {
		throw std::runtime_error("Failed to reopen credential file");
	}
	for (std::string line; std::getline(istr, line); ) {
		auto iter = line.begin();
		while (isspace(*iter)) {iter++;}
		if (*iter == '#') continue;
		if (*iter != '{' && *iter != '[' ) {
			// we have just a plain base64 token, not wrapped in JSON
			token = line;
			break;
		}
		rapidjson::Document doc;
{'fraction_non_alphanumeric': 0.1203847347192057, 'fraction_numerical': 0.006825938566552901, 'mean_word_length': 3.222707423580786, 'pattern_counts': {'":': 2, '<': 79, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1459690', 'n_tokens_mistral': 3316, 'n_tokens_neox': 3145, 'n_words': 974}