<commit_before>/*
 * External merge sorting of big file.
 *
 * WARNING: Works only when buffer size is aliquot part of file size!
 *          This is for simplicity of implementation.
 *
 * Copyright (c) 2015 <NAME> <<EMAIL>>
 * 
 * This project is licensed under the terms of the MIT license
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <time.h>

void print_arr(int A[], int n)
{
	int i;
	for (i = 0; i < n; i++)
		printf("%d\n", A[i]);
}

int compar(const void *p1, const void *p2)
{
	int *i1, *i2;

	i1 = (int *)p1;
	i2 = (int *)p2;

	return *i1 - *i2;
}

char *form_filename(int chunk, char *dir)
{
	char *path;
	int d;
	int dirlen, filelen, path_len;

	dirlen = strlen(dir);

	// File name is a chunk number, so
	// number of digits in chunk is a filename length
	filelen = 1;
	d = chunk;
	while (d)
	{
		d /= 10;
		filelen++;
	}	

	// One for '/' and one for '\0'
	path_len = dirlen + filelen + 1 + 1;
	path = malloc(path_len);
	snprintf(path, path_len, "%s/%d", dir, chunk);

	return path;
}

int save_buf(char *buf, int n, char *dir, int chunk)
{
	FILE *f = NULL;
	int nwrite;
	char *path;
	int ret = 0;

	path = form_filename(chunk, dir);
	f = fopen(path, "wb");
	if (f == NULL) {
		perror("fopen");
		ret = -1;
		goto exit;
	}

	nwrite = fwrite(buf, sizeof(int), n, f);
	if (nwrite != n) {
		perror("fwrite");
		ret = -1;
		goto exit;
	}

exit:
	if (f) {
		fclose(f);
	}

	if (path) {
		free(path);
	}


	return ret;
}

// Return chunks number of -1 on error
int split(FILE *f, off_t filesize, char *dir, char *buf, size_t bufsize)
{
	int chunks;
	int n, nread;
	int i;

	n = bufsize / sizeof(int);
   
	// XXX: we assume that filesize % bufsize == 0
	chunks = filesize / bufsize;
	for (i = 0; i < chunks; i++)
	{
		memset(buf, 0, bufsize);
		nread = fread(buf, sizeof(int), n, f);

		if (nread < n) {
			fprintf(stderr, "I/O error. Asked to read %d bytes, got %d\n", n, nread);
			return -1;
		}

		qsort(buf, nread, sizeof(int), compar);
		save_buf(buf, nread, dir, i);
		fprintf(stderr, "Saved chunk #%d of size %ld\n", i, nread * sizeof(int));
	}

	return chunks;
}

int merge(char *dir, char *buf, size_t bufsize, int chunks, size_t offset)
{
	FILE *f;
	int slice;
	char *path;
	int buf_offset;
	int i, n;

	slice = bufsize / chunks;
	buf_offset = 0;

	// Each chunk has `chunk` number of slices.
	for (i = 0; i < chunks; i++)
	{
		path = form_filename(i, dir);
		fprintf(stderr, "Chunk %s at offset %zu\n", path, offset);

		f = fopen(path, "rb");
		if (f == NULL) {
			perror("fopen");
			return -1;
		}
		fseek(f, offset, SEEK_SET);


		// Accumulate slices from each chunk in buffer.
		n = fread(buf + buf_offset, 1, slice, f);
		buf_offset += n;
	}

	// Thoughout this function we used buf as char array to use byte-addressing.
	// But now we need to act on ints inside that buffer, so we cast it to actual type.
	n = buf_offset / sizeof(int);
	qsort(buf, n, sizeof(int), compar);
	print_arr((int *)buf, n);

	return 0;
}<commit_msg>Close files, Sasha!<commit_after>/*
 * External merge sorting of big file.
 *
 * WARNING: Works only when buffer size is aliquot part of file size!
 *          This is for simplicity of implementation.
 *
 * Copyright (c) 2015 <NAME> <<EMAIL>>
 * 
 * This project is licensed under the terms of the MIT license
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <time.h>

void print_arr(int A[], int n)
{
	int i;
	for (i = 0; i < n; i++)
		printf("%d\n", A[i]);
}

int compar(const void *p1, const void *p2)
{
	int *i1, *i2;

	i1 = (int *)p1;
	i2 = (int *)p2;

	return *i1 - *i2;
}

char *form_filename(int chunk, char *dir)
{
	char *path;
	int d;
	int dirlen, filelen, path_len;

	dirlen = strlen(dir);

	// File name is a chunk number, so
	// number of digits in chunk is a filename length
	filelen = 1;
	d = chunk;
	while (d)
	{
		d /= 10;
		filelen++;
	}	

	// One for '/' and one for '\0'
	path_len = dirlen + filelen + 1 + 1;
	path = malloc(path_len);
	snprintf(path, path_len, "%s/%d", dir, chunk);

	return path;
}

int save_buf(char *buf, int n, char *dir, int chunk)
{
	FILE *f = NULL;
	int nwrite;
	char *path;
	int ret = 0;

	path = form_filename(chunk, dir);
	f = fopen(path, "wb");
	if (f == NULL) {
		perror("fopen");
		ret = -1;
		goto exit;
	}

	nwrite = fwrite(buf, sizeof(int), n, f);
	if (nwrite != n) {
		perror("fwrite");
		ret = -1;
		goto exit;
	}

exit:
	if (f) {
		fclose(f);
	}

	if (path) {
		free(path);
	}


	return ret;
}

// Return chunks number of -1 on error
int split(FILE *f, off_t filesize, char *dir, char *buf, size_t bufsize)
{
	int chunks;
	int n, nread;
	int i;

	n = bufsize / sizeof(int);
   
	// XXX: we assume that filesize % bufsize == 0
	chunks = filesize / bufsize;
	for (i = 0; i < chunks; i++)
	{
		memset(buf, 0, bufsize);
		nread = fread(buf, sizeof(int), n, f);

		if (nread < n) {
			fprintf(stderr, "I/O error. Asked to read %d bytes, got %d\n", n, nread);
			return -1;
		}

		qsort(buf, nread, sizeof(int), compar);
		save_buf(buf, nread, dir, i);
		fprintf(stderr, "Saved chunk #%d of size %ld\n", i, nread * sizeof(int));
	}

	return chunks;
}

int merge(char *dir, char *buf, size_t bufsize, int chunks, size_t offset)
{
	FILE *f;
	int slice;
	char *path;
	int buf_offset;
	int i, n;

	slice = bufsize / chunks;
	buf_offset = 0;

	// Each chunk has `chunk` number of slices.
	for (i = 0; i < chunks; i++)
	{
		path = form_filename(i, dir);
		fprintf(stderr, "Chunk %s at offset %zu\n", path, offset);

		f = fopen(path, "rb");
		if (f == NULL) {
			perror("fopen");
			return -1;
		}
		fseek(f, offset, SEEK_SET);


		// Accumulate slices from each chunk in buffer.
		n = fread(buf + buf_offset, 1, slice, f);
		buf_offset += n;
		fclose(f);
	}

	// Thoughout this function we used buf as char array to use byte-addressing.
	// But now we need to act on ints inside that buffer, so we cast it to actual type.
	n = buf_offset / sizeof(int);
	qsort(buf, n, sizeof(int), compar);
	print_arr((int *)buf, n);

	return 0;
}
{'fraction_non_alphanumeric': 0.1601376823471562, 'fraction_numerical': 0.011145713817406982, 'mean_word_length': 3.068, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '106886', 'n_tokens_mistral': 2628, 'n_tokens_neox': 2471, 'n_words': 829}