<commit_before>#include "file_accessor.h"
#include "s3.h"

using namespace himan;

buffer ReadFromLocalFile(const file_information& finfo)
{
	FILE* fp = fopen(finfo.file_location.c_str(), "rb");

	long offset = 0, length = 0;

	if (finfo.offset && finfo.length)
	{
		offset = finfo.offset.get();
		length = finfo.length.get();
		fseek(fp, offset, SEEK_SET);
	}
	else
	{
		// read whole file
		fseek(fp, 0, SEEK_END);
		length = ftell(fp);
		rewind(fp);
	}

	buffer buf;
	buf.data = new unsigned char[length];
	buf.length = length;

	fread(buf.data, buf.length, 1, fp);<commit_msg>match free() with malloc()<commit_after>#include "file_accessor.h"
#include "s3.h"

using namespace himan;

buffer ReadFromLocalFile(const file_information& finfo)
{
	FILE* fp = fopen(finfo.file_location.c_str(), "rb");

	long offset = 0, length = 0;

	if (finfo.offset && finfo.length)
	{
		offset = finfo.offset.get();
		length = finfo.length.get();
		fseek(fp, offset, SEEK_SET);
	}
	else
	{
		// read whole file
		fseek(fp, 0, SEEK_END);
		length = ftell(fp);
		rewind(fp);
	}

	buffer buf;
	buf.data = reinterpret_cast<unsigned char*>(malloc(length));
	buf.length = length;

	fread(buf.data, buf.length, 1, fp);
{'fraction_non_alphanumeric': 0.15802675585284282, 'fraction_numerical': 0.008361204013377926, 'mean_word_length': 4.181818181818182, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1537879', 'n_tokens_mistral': 505, 'n_tokens_neox': 469, 'n_words': 105}