<commit_before>#ifndef _REMOTELISTOFFILES_H
#define _REMOTELISTOFFILES_H

#include <iostream>
using namespace std;

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <errno.h>
#include "amazonCredentials.h"
#include "threads.h"


class RemoteListOfFiles
{
private:
	AmazonCredentials *amazonCredentials;
	uint32_t allocCount;
	Threads *threads;
	int failed;

	static char *extractMd5FromEtag(char *etag);
	static uint32_t extractMtimeFromHeaders(char *headers);

	int parseListOfFiles(char *body, uint64_t bodySize, uint8_t *isTruncated, char *lastKey, char *errorResult);
	int performGetOnBucket(char *url, char *marker, int setLocationHeader, char *body, uint64_t *bodySize, uint32_t *statusCode, char *errorResult);
	int performHeadOnFile(char *url, char *remotePath, uint32_t *remoteMtime, uint32_t *statusCode, char *errorResult);
	int performPutOnBucket(char *url, char *region, uint32_t *statusCode, char *errorResult);

public:
	RemoteListOfFiles(AmazonCredentials *amazonCredentials);
	~RemoteListOfFiles();
	
	void add(char *path, char *md5);
	int downloadList();
	int resolveMtimes();

	int checkAuth();
	int createBucket(char *region);

	void runOverThread(int threadNumber, int pos);

	char **paths;
	char **md5s;
	uint32_t *mtimes;
	uint32_t count;
	int showProgress;
	int useSsl;
	int connectTimeout;
	int networkTimeout;
};

#define LIST_FAILED 0
#define LIST_SUCCESS 1

#define HEAD_FAILED 0
#define HEAD_SUCCESS 1

#define CREATE_FAILED 0
#define CREATE_SUCCESS 1

#define AUTH_FAILED 0
#define AUTH_FAILED_BUCKET_DOESNT_EXISTS -1
#define AUTH_SUCCESS 1

int checkAuth(AmazonCredentials *amazonCredentials);

#endif<commit_msg>garbage<commit_after>#ifndef _REMOTELISTOFFILES_H
#define _REMOTELISTOFFILES_H

#include <iostream>
using namespace std;

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <errno.h>
#include "amazonCredentials.h"
#include "threads.h"


class RemoteListOfFiles
{
private:
	AmazonCredentials *amazonCredentials;
	uint32_t allocCount;
	Threads *threads;
	int failed;

	static char *extractMd5FromEtag(char *etag);
	static uint32_t extractMtimeFromHeaders(char *headers);

	int parseListOfFiles(char *body, uint64_t bodySize, uint8_t *isTruncated, char *lastKey, char *errorResult);
	int performGetOnBucket(char *url, char *marker, int setLocationHeader, char *body, uint64_t *bodySize, uint32_t *statusCode, char *errorResult);
	int performHeadOnFile(char *url, char *remotePath, uint32_t *remoteMtime, uint32_t *statusCode, char *errorResult);
	int performPutOnBucket(char *url, char *region, uint32_t *statusCode, char *errorResult);

public:
	RemoteListOfFiles(AmazonCredentials *amazonCredentials);
	~RemoteListOfFiles();
	
	void add(char *path, char *md5);
	int downloadList();
	int resolveMtimes();

	int checkAuth();
	int createBucket(char *region);

	void runOverThread(int threadNumber, int pos);

	char **paths;
	char **md5s;
	uint32_t *mtimes;
	uint32_t count;
	int showProgress;
	int useSsl;
	int connectTimeout;
	int networkTimeout;
};

#define LIST_FAILED 0
#define LIST_SUCCESS 1

#define HEAD_FAILED 0
#define HEAD_SUCCESS 1

#define CREATE_FAILED 0
#define CREATE_SUCCESS 1

#define AUTH_FAILED 0
#define AUTH_FAILED_BUCKET_DOESNT_EXISTS -1
#define AUTH_SUCCESS 1

#endif
{'fraction_non_alphanumeric': 0.09800362976406533, 'fraction_numerical': 0.019963702359346643, 'mean_word_length': 6.892601431980907, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4192118', 'n_tokens_mistral': 1211, 'n_tokens_neox': 1159, 'n_words': 312}