<commit_before>#include "pstore/mmap-window.h"
#include "pstore/compat.h"
#include "pstore/die.h"

#include <sys/types.h>
#include <sys/mman.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <unistd.h>

#define PAGE_SIZE		getpagesize()
#define PAGE_MASK		(~(PAGE_SIZE-1))

/*
 * The minimum mmap sliding window size is two pages. That's because when we
 * move the sliding window we need to align the starting offset at page
 * boundary.
 */
#define MMAP_WINDOW_LEN		(128LL * 1024LL * 1024LL)	/* 128 MiB */

static struct mmap_window *mmap_window__new(void)
{
	struct mmap_window *self = calloc(sizeof(*self), 1);

	if (!self)
		die("out of memory");

	return self;
}

static void mmap_window__delete(struct mmap_window *self)
{
	free(self);
}

struct mmap_window *mmap_window__map(int fd, off64_t offset, off64_t length)
{
	struct mmap_window *self = mmap_window__new();
	size_t mmap_len;

	mmap_len = length;
	if (mmap_len > MMAP_WINDOW_LEN)
		mmap_len = MMAP_WINDOW_LEN;

	self->mmap = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, offset);
	if (self->mmap == MAP_FAILED)
		die("mmap");

	self->mmap_end	= self->mmap + mmap_len;
	self->mmap_len	= mmap_len;
	self->length	= length;
	self->offset	= offset;
	self->fd	= fd;

	return self;
}

void mmap_window__unmap(struct mmap_window *self)
{
	if (munmap(self->mmap, self->mmap_len) < 0)
		die("munmap");

	mmap_window__delete(self);
}

void *mmap_window__start(struct mmap_window *self)
{
	return self->mmap + self->mmap_pos;
}

void *mmap_window__slide(struct mmap_window *self, void *p)
{
	off64_t window_start;
	size_t remaining;
	size_t mmap_pos;
	uint64_t pos;

	mmap_pos = p - self->mmap;

	pos = self->pos + mmap_pos; 

	remaining = self->length - pos - 1;
	if (remaining == 0)
		return NULL;

	if (munmap(self->mmap, self->mmap_len) < 0)
		die("munmap");

	self->mmap_len = MMAP_WINDOW_LEN;
	if (self->mmap_len > remaining)
		self->mmap_len = remaining;

	window_start = pos & PAGE_MASK;

	self->mmap = mmap(NULL, self->mmap_len, PROT_READ, MAP_PRIVATE, self->fd, self->offset + window_start);
	if (self->mmap == MAP_FAILED)
		die("mmap");

	self->mmap_end	= self->mmap + self->mmap_len;
	self->mmap_pos	= pos - window_start;
	self->pos	= pos;

	return mmap_window__start(self);
}<commit_msg>builtin-import: Use POSIX_MADV_SEQUENTIAL<commit_after>#include "pstore/mmap-window.h"
#include "pstore/compat.h"
#include "pstore/die.h"

#include <sys/types.h>
#include <sys/mman.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>

#define PAGE_SIZE		getpagesize()
#define PAGE_MASK		(~(PAGE_SIZE-1))

/*
 * The minimum mmap sliding window size is two pages. That's because when we
 * move the sliding window we need to align the starting offset at page
 * boundary.
 */
#define MMAP_WINDOW_LEN		(128LL * 1024LL * 1024LL)	/* 128 MiB */

static struct mmap_window *mmap_window__new(void)
{
	struct mmap_window *self = calloc(sizeof(*self), 1);

	if (!self)
		die("out of memory");

	return self;
}

static void mmap_window__delete(struct mmap_window *self)
{
	free(self);
}

struct mmap_window *mmap_window__map(int fd, off64_t offset, off64_t length)
{
	struct mmap_window *self = mmap_window__new();
	size_t mmap_len;

	mmap_len = length;
	if (mmap_len > MMAP_WINDOW_LEN)
		mmap_len = MMAP_WINDOW_LEN;

	self->mmap = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, offset);
	if (self->mmap == MAP_FAILED)
		die("mmap");

	self->mmap_end	= self->mmap + mmap_len;
	self->mmap_len	= mmap_len;
	self->length	= length;
	self->offset	= offset;
	self->fd	= fd;

	if (posix_madvise(self->mmap, self->mmap_len, POSIX_MADV_SEQUENTIAL) < 0)
		die("posix_madvise");

	return self;
}

void mmap_window__unmap(struct mmap_window *self)
{
	if (munmap(self->mmap, self->mmap_len) < 0)
		die("munmap");

	mmap_window__delete(self);
}

void *mmap_window__start(struct mmap_window *self)
{
	return self->mmap + self->mmap_pos;
}

void *mmap_window__slide(struct mmap_window *self, void *p)
{
	off64_t window_start;
	size_t remaining;
	size_t mmap_pos;
	uint64_t pos;

	mmap_pos = p - self->mmap;

	pos = self->pos + mmap_pos; 

	remaining = self->length - pos - 1;
	if (remaining == 0)
		return NULL;

	if (munmap(self->mmap, self->mmap_len) < 0)
		die("munmap");

	self->mmap_len = MMAP_WINDOW_LEN;
	if (self->mmap_len > remaining)
		self->mmap_len = remaining;

	window_start = pos & PAGE_MASK;

	self->mmap = mmap(NULL, self->mmap_len, PROT_READ, MAP_PRIVATE, self->fd, self->offset + window_start);
	if (self->mmap == MAP_FAILED)
		die("mmap");

	self->mmap_end	= self->mmap + self->mmap_len;
	self->mmap_pos	= pos - window_start;
	self->pos	= pos;

	if (posix_madvise(self->mmap, self->mmap_len, POSIX_MADV_SEQUENTIAL) < 0)
		die("posix_madvise");

	return mmap_window__start(self);
}
{'fraction_non_alphanumeric': 0.14116177389131793, 'fraction_numerical': 0.012075785967103894, 'mean_word_length': 4.671782762691854, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4499422', 'n_tokens_mistral': 2093, 'n_tokens_neox': 1976, 'n_words': 432}