<commit_before>// JAB: this is required for pwrite() in this module
#undef _XOPEN_SOURCE
#define _XOPEN_SOURCE 500

#include "gb-include.h"

#include "BigFile.h"
#include "Dir.h"
#include "Threads.h"
#include "Stats.h"
#include "Statsdb.h"
#include "DiskPageCache.h"

#ifdef ASYNCIO
#include <aio.h>
#endif

// main.cpp will wait for this to be zero before exiting so all unlink/renames
// can complete
int32_t g_unlinkRenameThreads = 0;

int64_t g_lastDiskReadStarted = 0LL;
int64_t g_lastDiskReadCompleted = 0LL;
bool      g_diskIsStuck = false;

static void  doneWrapper        ( void *state , ThreadEntry *t ) ;
static bool  readwrite_r        ( FileState *fstate , ThreadEntry *t ) ;

BigFile::~BigFile () {
	close();
}

//#define O_DIRECT 040000

BigFile::BigFile () {
	m_permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH ;
	m_flags       = O_RDWR ; // | O_DIRECT;
	// NULLify all ptrs to files
	//for ( int32_t i = 0 ; i < MAX_PART_FILES ; i++ ) m_files[i] = NULL;
	m_maxParts = 0;
	m_numParts = 0;
	m_pc  = NULL;
	m_vfd = -1;
	//m_vfdAllowed = false;
	m_fileSize = -1;
	m_lastModified = -1;
	m_numThreads = 0;
	m_isClosing = false;
	g_lastDiskReadStarted = 0;
	g_lastDiskReadCompleted = 0;
	g_diskIsStuck = false;
}

// we alternate parts into "dirname" and "stripeDir"
// . return false and set g_errno on error
bool BigFile::set ( char *dir , char *baseFilename , char *stripeDir ) {
	// reset filsize
	m_fileSize = -1;
	m_lastModified = -1;
	// m_baseFilename contains the "dir" in it
	//sprintf(m_baseFilename ,"%s/%s", dirname  , baseFilename );

	m_dir.reset();
	m_baseFilename.reset();

	m_dir         .setLabel("bfd");
	m_baseFilename.setLabel("bfbf");

	if ( ! m_dir.safeStrcpy          ( dir          ) ) return false;
	if ( ! m_baseFilename.safeStrcpy ( baseFilename ) ) return false;

	//strcpy ( m_baseFilename , baseFilename  );
	//strcpy ( m_dir          , dir           );
	//if ( stripeDir ) strcpy ( m_stripeDir    , stripeDir     );
	//else             m_stripeDir[0] = '\0';
	// reset # of parts
	m_numParts = 0;
	m_maxParts = 0;
	// now add parts from both directories
	if ( ! addParts ( dir       ) ) return false;
	//if ( ! addParts ( m_stripeDir ) ) return false;
	return true;
}

bool BigFile::reset ( ) {
	// reset filsize
	m_fileSize = -1;
	m_lastModified = -1;
	// m_baseFilename contains the "dir" in it
	//sprintf(m_baseFilename ,"%s/%s", dirname  , baseFilename );
	//strcpy ( m_baseFilename , baseFilename  );
	//strcpy ( m_dir          , dir           );
	//if ( stripeDir ) strcpy ( m_stripeDir    , stripeDir     );
	//else             m_stripeDir[0] = '\0';
	// reset # of parts
	m_numParts = 0;
	m_maxParts = 0;
	// now add parts from both directories
	if ( ! addParts ( m_dir.getBufStart() ) ) return false;
	//if ( ! addParts ( m_stripeDir ) ) return false;
	return true;
}
	

bool BigFile::addParts ( char *dirname ) {
	// if dirname is NULL return true
	if ( ! dirname[0] ) return true;
	// . now set the names of all the Files that we consist of
	// . get the directory entry and find out what parts we have
	Dir dir;
	dir.set ( dirname );
	// set our directory class
	if (!dir.open()) return log("disk: openDir (\"%s\") failed",dirname);
	// match files with this pattern in the directory
	char pattern[256];
	sprintf(pattern,"%s*", m_baseFilename.getBufStart() );
	// length of the base filename
	int32_t blen = gbstrlen ( m_baseFilename.getBufStart() );
	// . set our m_files array
	// . addFile() will return false on problems<commit_msg>fix little core<commit_after>// JAB: this is required for pwrite() in this module
#undef _XOPEN_SOURCE
#define _XOPEN_SOURCE 500

#include "gb-include.h"

#include "BigFile.h"
#include "Dir.h"
#include "Threads.h"
#include "Stats.h"
#include "Statsdb.h"
#include "DiskPageCache.h"

#ifdef ASYNCIO
#include <aio.h>
#endif

// main.cpp will wait for this to be zero before exiting so all unlink/renames
// can complete
int32_t g_unlinkRenameThreads = 0;

int64_t g_lastDiskReadStarted = 0LL;
int64_t g_lastDiskReadCompleted = 0LL;
bool      g_diskIsStuck = false;

static void  doneWrapper        ( void *state , ThreadEntry *t ) ;
static bool  readwrite_r        ( FileState *fstate , ThreadEntry *t ) ;

BigFile::~BigFile () {
	close();
}

//#define O_DIRECT 040000

BigFile::BigFile () {
	m_permissions = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH ;
	m_flags       = O_RDWR ; // | O_DIRECT;
	// NULLify all ptrs to files
	//for ( int32_t i = 0 ; i < MAX_PART_FILES ; i++ ) m_files[i] = NULL;
	m_maxParts = 0;
	m_numParts = 0;
	m_pc  = NULL;
	m_vfd = -1;
	//m_vfdAllowed = false;
	m_fileSize = -1;
	m_lastModified = -1;
	m_numThreads = 0;
	m_isClosing = false;
	g_lastDiskReadStarted = 0;
	g_lastDiskReadCompleted = 0;
	g_diskIsStuck = false;
}

// we alternate parts into "dirname" and "stripeDir"
// . return false and set g_errno on error
bool BigFile::set ( char *dir , char *baseFilename , char *stripeDir ) {
	// reset filsize
	m_fileSize = -1;
	m_lastModified = -1;
	// m_baseFilename contains the "dir" in it
	//sprintf(m_baseFilename ,"%s/%s", dirname  , baseFilename );

	m_dir.reset();
	m_baseFilename.reset();

	m_dir         .setLabel("bfd");
	m_baseFilename.setLabel("bfbf");

	if ( ! m_dir.safeStrcpy          ( dir          ) ) return false;
	if ( ! m_baseFilename.safeStrcpy ( baseFilename ) ) return false;

	//strcpy ( m_baseFilename , baseFilename  );
	//strcpy ( m_dir          , dir           );
	//if ( stripeDir ) strcpy ( m_stripeDir    , stripeDir     );
	//else             m_stripeDir[0] = '\0';
	// reset # of parts
	m_numParts = 0;
	m_maxParts = 0;
	// now add parts from both directories
	if ( ! addParts ( dir       ) ) return false;
	//if ( ! addParts ( m_stripeDir ) ) return false;
	return true;
}

bool BigFile::reset ( ) {
	// reset filsize
	m_fileSize = -1;
	m_lastModified = -1;
	// m_baseFilename contains the "dir" in it
	//sprintf(m_baseFilename ,"%s/%s", dirname  , baseFilename );
	//strcpy ( m_baseFilename , baseFilename  );
	//strcpy ( m_dir          , dir           );
	//if ( stripeDir ) strcpy ( m_stripeDir    , stripeDir     );
	//else             m_stripeDir[0] = '\0';
	// reset # of parts
	m_numParts = 0;
	m_maxParts = 0;
	// now add parts from both directories
	if ( ! addParts ( m_dir.getBufStart() ) ) return false;
	//if ( ! addParts ( m_stripeDir ) ) return false;
	return true;
}
	

bool BigFile::addParts ( char *dirname ) {
	// if dirname is NULL return true
	if ( ! dirname || ! dirname[0] ) return true;
	// . now set the names of all the Files that we consist of
	// . get the directory entry and find out what parts we have
	Dir dir;
	dir.set ( dirname );
	// set our directory class
	if (!dir.open()) return log("disk: openDir (\"%s\") failed",dirname);
	// match files with this pattern in the directory
	char pattern[256];
	sprintf(pattern,"%s*", m_baseFilename.getBufStart() );
	// length of the base filename
	int32_t blen = gbstrlen ( m_baseFilename.getBufStart() );
	// . set our m_files array
	// . addFile() will return false on problems
{'fraction_non_alphanumeric': 0.1277231952157198, 'fraction_numerical': 0.013384593478570411, 'mean_word_length': 3.0414269275028767, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7002009', 'n_tokens_mistral': 2645, 'n_tokens_neox': 2570, 'n_words': 722}