<commit_before>/*©mit**************************************************************************
*                                                                              *
* This file is part of FRIEND UNIFYING PLATFORM.                               *
* Copyright (c) Friend Software Labs AS. All rights reserved.                  *
*                                                                              *
* Licensed under the Source EULA. Please refer to the copy of the MIT License, *
* found in the file license_mit.txt.                                           *
*                                                                              *
*****************************************************************************©*/
/** @file
 * 
 * file contain functitons related to local files
 *
 *  @author PS (<NAME>)
 *  @date created 2016
 */

#define _DEFAULT_SOURCE 1 //required for mmap flags

#include <sys/types.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <network/locfile.h>
#include <util/string.h>
#include <util/buffered_string.h>
#include <util/log/log.h>

#include <ftw.h>
#include <unistd.h>
#include <sys/statvfs.h>
#include <util/murmurhash3.h>
#include <errno.h>

#if LOCFILE_USE_MMAP == 1
#include <sys/mman.h>
#endif

/**
 * Get filename from path
 *
 * @param path pointer to char table with path
 * @param len length of provided path
 * @return function returns pointer to char from which filename is started
 */
static inline char *GetFileNamePtr( char *path, int len )
{
	int i = len;
	while( i != 0 )
	{
		if( path[ i ] == '/' )
		{
			return &path[ i+1 ];
		}
		i--;
	}
	return path;
}

#if LOCFILE_USE_MMAP == 0
/**
 * Read a block of memory from LocFile
 *
 * @param file pointer to LocFile from which data will be readed
 * @param offset number of bytes from which data will be readed
 * @param size number of bytes to read
 * @return number of bytes readed from file
 */
static inline int LocFileRead( LocFile* file, FILE *fp, long long offset, long long size )
{
	if( file == NULL )
	{
		FERROR("Cannot read file which doesnt exist!\n");
		return -1;
	}

	file->lf_Buffer = (char *)FMalloc( size + 1 );
	if( file->lf_Buffer == NULL )
	{
		DEBUG("Cannot allocate memory for file\n");
		return 0;
	}
	
	file->lf_FileSize = size;
	fseek( fp, offset, SEEK_SET );
	int result = fread( file->lf_Buffer, size, 1, fp );
	if( result < size )
	{
		return result; 
	}
	return 0;
}
#endif

/**
 * Create new LocFile structure and read file from provided path
 *
 * @param path pointer to char table with path
 * @param flags additional flags used to open file
 * @return pointer to new LocFile when success, otherwise NULL
 */
LocFile* LocFileNew( char* path, unsigned int flags )
{
	if( path == NULL )
	{
		FERROR("File path is null\n");
		return NULL;
	}
	FILE* fp = fopen( path, "rb" );
	if( fp == NULL )
	{
		Log( FLOG_ERROR, "Cannot open file %s, errno: %s\n", path, strerror(errno) );
		return NULL;
	}
	
	struct stat st;
	if( stat( path, &st ) < 0 )
	{
		FERROR( "Cannot stat file: '%s'.\n", path );
		fclose( fp );
		return NULL;
	}
	
	LocFile* fo = (LocFile*) FCalloc( 1, sizeof(LocFile) );
	if( fo != NULL )
	{
		fo->lf_PathLength = strlen( path );
		fo->lf_Path = StringDuplicateN( path, fo->lf_PathLength );
		//fo->lf_Filename = StringDuplicate( GetFileNamePtr( path, fo->lf_PathLength ) );
		
		MURMURHASH3( fo->lf_Path, fo->lf_PathLength, fo->hash );
		
		memcpy(  &(fo->lf_Info),  &st, sizeof( struct stat) );

		fseek( fp, 0, SEEK_END );
		long fsize = ftell( fp );
		fseek( fp, 0, SEEK_SET );  //same as rewind(f);
		fo->lf_FileSize = fsize;// st.st_size; //ftell( fp );

		if( flags & FILE_READ_NOW )
		{
#if LOCFILE_USE_MMAP == 0
			LocFileRead( fo, fp, 0, fo->lf_FileSize );
#else

			fo->lf_Buffer = mmap(NULL/*address can be anywhere*/,
					fo->lf_FileSize/*map whole file*/,<commit_msg>FIX - check root "/" paths while loading as files<commit_after>/*©mit**************************************************************************
*                                                                              *
* This file is part of FRIEND UNIFYING PLATFORM.                               *
* Copyright (c) Friend Software Labs AS. All rights reserved.                  *
*                                                                              *
* Licensed under the Source EULA. Please refer to the copy of the MIT License, *
* found in the file license_mit.txt.                                           *
*                                                                              *
*****************************************************************************©*/
/** @file
 * 
 * file contain functitons related to local files
 *
 *  @author PS (<NAME>)
 *  @date created 2016
 */

#define _DEFAULT_SOURCE 1 //required for mmap flags

#include <sys/types.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <network/locfile.h>
#include <util/string.h>
#include <util/buffered_string.h>
#include <util/log/log.h>

#include <ftw.h>
#include <unistd.h>
#include <sys/statvfs.h>
#include <util/murmurhash3.h>
#include <errno.h>

#if LOCFILE_USE_MMAP == 1
#include <sys/mman.h>
#endif

/**
 * Get filename from path
 *
 * @param path pointer to char table with path
 * @param len length of provided path
 * @return function returns pointer to char from which filename is started
 */
static inline char *GetFileNamePtr( char *path, int len )
{
	int i = len;
	while( i != 0 )
	{
		if( path[ i ] == '/' )
		{
			return &path[ i+1 ];
		}
		i--;
	}
	return path;
}

#if LOCFILE_USE_MMAP == 0
/**
 * Read a block of memory from LocFile
 *
 * @param file pointer to LocFile from which data will be readed
 * @param offset number of bytes from which data will be readed
 * @param size number of bytes to read
 * @return number of bytes readed from file
 */
static inline int LocFileRead( LocFile* file, FILE *fp, long long offset, long long size )
{
	if( file == NULL )
	{
		FERROR("Cannot read file which doesnt exist!\n");
		return -1;
	}

	file->lf_Buffer = (char *)FMalloc( size + 1 );
	if( file->lf_Buffer == NULL )
	{
		DEBUG("Cannot allocate memory for file\n");
		return 0;
	}
	
	file->lf_FileSize = size;
	fseek( fp, offset, SEEK_SET );
	int result = fread( file->lf_Buffer, size, 1, fp );
	if( result < size )
	{
		return result; 
	}
	return 0;
}
#endif

/**
 * Create new LocFile structure and read file from provided path
 *
 * @param path pointer to char table with path
 * @param flags additional flags used to open file
 * @return pointer to new LocFile when success, otherwise NULL
 */
LocFile* LocFileNew( char* path, unsigned int flags )
{
	if( path == NULL )
	{
		FERROR("File path is null\n");
		return NULL;
	}
	FILE* fp = fopen( path, "rb" );
	if( fp == NULL )
	{
		Log( FLOG_ERROR, "Cannot open file %s, errno: %s\n", path, strerror(errno) );
		return NULL;
	}
	
	struct stat st;
	if( stat( path, &st ) < 0 )
	{
		FERROR( "Cannot stat file: '%s'.\n", path );
		fclose( fp );
		return NULL;
	}
	
	if( S_ISDIR( st.st_mode ) )
	{
		FERROR( "'%s' is a directory. Can not open.\n", path );
		fclose( fp );
		return NULL;
	}
	
	LocFile* fo = (LocFile*) FCalloc( 1, sizeof(LocFile) );
	if( fo != NULL )
	{
		fo->lf_PathLength = strlen( path );
		fo->lf_Path = StringDuplicateN( path, fo->lf_PathLength );
		//fo->lf_Filename = StringDuplicate( GetFileNamePtr( path, fo->lf_PathLength ) );
		
		MURMURHASH3( fo->lf_Path, fo->lf_PathLength, fo->hash );
		
		memcpy(  &(fo->lf_Info),  &st, sizeof( struct stat) );

		fseek( fp, 0, SEEK_END );
		long fsize = ftell( fp );
		fseek( fp, 0, SEEK_SET );  //same as rewind(f);
		fo->lf_FileSize = fsize;// st.st_size; //ftell( fp );

		if( flags & FILE_READ_NOW )
		{
#if LOCFILE_USE_MMAP == 0
			LocFileRead( fo, fp, 0, fo->lf_FileSize );
#else

			fo->lf_Buffer = mmap(NULL/*address can be anywhere*/,
					fo->lf_FileSize/*map whole file*/,
{'fraction_non_alphanumeric': 0.16410965794768612, 'fraction_numerical': 0.005533199195171026, 'mean_word_length': 2.5425389755011136, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6680976', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2547, 'n_words': 850}