<commit_before>#if defined(HAVE_LEPTONICA) && defined(HAVE_TESSERACT)

#include "tesseract/baseapi.h"
#include "tesseract/genericvector.h"
#include "tesseract/serialis.h"
#include "tesseract/strngs.h"

extern "C" {

#include "allheaders.h"

#include "tessocr.h"

/* Non-overridden leptonica alloc functions. These should never
 * actually be used. */
void *leptonica_malloc(size_t blocksize)
{
	void *ret = malloc(blocksize);
#ifdef DEBUG_ALLOCS
	printf("%d LEPTONICA_MALLOC %d -> %p\n", event++, (int)blocksize, ret);
	fflush(stdout);
#endif
	return ret;
}

void *leptonica_calloc(size_t numelm, size_t elemsize)
{
	void *ret = calloc(numelm, elemsize);
#ifdef DEBUG_ALLOCS
	printf("%d LEPTONICA_CALLOC %d,%d -> %p\n", event++, (int)numelm, (int)elem> fflush(stdout);
#endif
	return ret;
}

void *leptonica_realloc(void *ptr, size_t blocksize)
{
	void *ret = realloc(ptr, blocksize);
#ifdef DEBUG_ALLOCS
	printf("%d LEPTONICA_REALLOC %p,%d -> %p\n", event++, ptr, (int)blocksize, ret);
	fflush(stdout);
#endif
	return ret;
}

void leptonica_free(void *ptr)
{
#ifdef DEBUG_ALLOCS
	printf("%d LEPTONICA_FREE %p\n", event++, ptr);
	fflush(stdout);
#endif
	free(ptr);
}

/* Now the actual allocations to be used for leptonica. Unfortunately
 * we have to use a nasty global here. */
static fz_context *leptonica_mem;

static void *my_leptonica_malloc(size_t size)
{
	void *ret = Memento_label(fz_malloc_no_throw(leptonica_mem, size), "leptonica_malloc");
#ifdef DEBUG_ALLOCS
	printf("%d MY_LEPTONICA_MALLOC(%p) %d -> %p\n", event++, leptonica_mem, (int)size, ret);
	fflush(stdout);
#endif
	return ret;
}

static void my_leptonica_free(void *ptr)
{
#ifdef DEBUG_ALLOCS
	printf("%d MY_LEPTONICA_FREE(%p) %p\n", event++, leptonica_mem, ptr);
	fflush(stdout);
#endif
	fz_free(leptonica_mem, ptr);
}

static bool
load_file(const char* filename, GenericVector<char>* data)
{
	bool result = false;
	FILE *fp = fopen(filename, "rb");
	if (fp == NULL)
		return false;

	fseek(fp, 0, SEEK_END);
	long size = ftell(fp);
	fseek(fp, 0, SEEK_SET);

	// Trying to open a directory on Linux sets size to LONG_MAX. Catch it here.
	if (size > 0 && size < LONG_MAX)
	{
		// reserve an extra byte in case caller wants to append a '\0' character
		data->reserve(size + 1);
		data->resize_no_init(size);
		result = static_cast<long>(fread(&(*data)[0], 1, size, fp)) == size;
	}
	fclose(fp);
	return result;
}

static bool
tess_file_reader(const char *fname, GenericVector<char> *out)
{
	//const char *file = fname;
	//const char *s;

	//for (s = fname; *s; s++)
	//	if (*s == '\\' || *s == '/')
	//		file = s+1;

	/* FIXME: Look for inbuilt ones. */

	/* Then under TESSDATA */
	return load_file(fname, out);
}

static void
set_leptonica_mem(fz_context *ctx)
{
	int die = 0;

	fz_lock(ctx, FZ_LOCK_ALLOC);
	die = (leptonica_mem != NULL);
	if (!die)
		leptonica_mem = ctx;
	fz_unlock(ctx, FZ_LOCK_ALLOC);
	if (die)
		fz_throw(ctx, FZ_ERROR_GENERIC, "Attempt to use Tesseract from 2 threads at once!");
}

static void
clear_leptonica_mem(fz_context *ctx)
{
	int die = 0;

	fz_lock(ctx, FZ_LOCK_ALLOC);
	die = (leptonica_mem == NULL);
	if (!die)
		leptonica_mem = NULL;
	fz_unlock(ctx, FZ_LOCK_ALLOC);
	if (die)
		fz_throw(ctx, FZ_ERROR_GENERIC, "Attempt to use Tesseract from 2 threads at once!");
}

void *ocr_init(fz_context *ctx, const char *language)
{
	tesseract::TessBaseAPI *api;

	set_leptonica_mem(ctx);
	setPixMemoryManager(my_leptonica_malloc, my_leptonica_free);
	api = new tesseract::TessBaseAPI();

	if (api == NULL)
	{
		clear_leptonica_mem(ctx);
		setPixMemoryManager(malloc, free);
		fz_throw(ctx, FZ_ERROR_GENERIC, "Tesseract initialisation failed");
	}

	// Initialize tesseract-ocr with English, without specifying tessdata path
	if (api->Init(NULL, 0, /* data, data_size */
		language,
		tesseract::OcrEngineMode::OEM_DEFAULT,
		NULL, 0, /* configs, configs_size */
		NULL, NULL, /* vars_vec */
		false, /* set_only_non_debug_params */
		&tess_file_reader))
	{
		delete api;
		clear_leptonica_mem(ctx);
		setPixMemoryManager(malloc, free);
		fz_throw(ctx, FZ_ERROR_GENERIC, "Tesseract initialisation failed");
	}

	return api;
}

void ocr_fin(fz_context *ctx, void *api_)
{
	tesseract::TessBaseAPI *api = (tesseract::TessBaseAPI *)api_;
<commit_msg>Ensure ocr_init defaults to 'eng' if no language string passed.<commit_after>#if defined(HAVE_LEPTONICA) && defined(HAVE_TESSERACT)

#include "tesseract/baseapi.h"
#include "tesseract/genericvector.h"
#include "tesseract/serialis.h"
#include "tesseract/strngs.h"

extern "C" {

#include "allheaders.h"

#include "tessocr.h"

/* Non-overridden leptonica alloc functions. These should never
 * actually be used. */
void *leptonica_malloc(size_t blocksize)
{
	void *ret = malloc(blocksize);
#ifdef DEBUG_ALLOCS
	printf("%d LEPTONICA_MALLOC %d -> %p\n", event++, (int)blocksize, ret);
	fflush(stdout);
#endif
	return ret;
}

void *leptonica_calloc(size_t numelm, size_t elemsize)
{
	void *ret = calloc(numelm, elemsize);
#ifdef DEBUG_ALLOCS
	printf("%d LEPTONICA_CALLOC %d,%d -> %p\n", event++, (int)numelm, (int)elem> fflush(stdout);
#endif
	return ret;
}

void *leptonica_realloc(void *ptr, size_t blocksize)
{
	void *ret = realloc(ptr, blocksize);
#ifdef DEBUG_ALLOCS
	printf("%d LEPTONICA_REALLOC %p,%d -> %p\n", event++, ptr, (int)blocksize, ret);
	fflush(stdout);
#endif
	return ret;
}

void leptonica_free(void *ptr)
{
#ifdef DEBUG_ALLOCS
	printf("%d LEPTONICA_FREE %p\n", event++, ptr);
	fflush(stdout);
#endif
	free(ptr);
}

/* Now the actual allocations to be used for leptonica. Unfortunately
 * we have to use a nasty global here. */
static fz_context *leptonica_mem;

static void *my_leptonica_malloc(size_t size)
{
	void *ret = Memento_label(fz_malloc_no_throw(leptonica_mem, size), "leptonica_malloc");
#ifdef DEBUG_ALLOCS
	printf("%d MY_LEPTONICA_MALLOC(%p) %d -> %p\n", event++, leptonica_mem, (int)size, ret);
	fflush(stdout);
#endif
	return ret;
}

static void my_leptonica_free(void *ptr)
{
#ifdef DEBUG_ALLOCS
	printf("%d MY_LEPTONICA_FREE(%p) %p\n", event++, leptonica_mem, ptr);
	fflush(stdout);
#endif
	fz_free(leptonica_mem, ptr);
}

static bool
load_file(const char* filename, GenericVector<char>* data)
{
	bool result = false;
	FILE *fp = fopen(filename, "rb");
	if (fp == NULL)
		return false;

	fseek(fp, 0, SEEK_END);
	long size = ftell(fp);
	fseek(fp, 0, SEEK_SET);

	// Trying to open a directory on Linux sets size to LONG_MAX. Catch it here.
	if (size > 0 && size < LONG_MAX)
	{
		// reserve an extra byte in case caller wants to append a '\0' character
		data->reserve(size + 1);
		data->resize_no_init(size);
		result = static_cast<long>(fread(&(*data)[0], 1, size, fp)) == size;
	}
	fclose(fp);
	return result;
}

static bool
tess_file_reader(const char *fname, GenericVector<char> *out)
{
	//const char *file = fname;
	//const char *s;

	//for (s = fname; *s; s++)
	//	if (*s == '\\' || *s == '/')
	//		file = s+1;

	/* FIXME: Look for inbuilt ones. */

	/* Then under TESSDATA */
	return load_file(fname, out);
}

static void
set_leptonica_mem(fz_context *ctx)
{
	int die = 0;

	fz_lock(ctx, FZ_LOCK_ALLOC);
	die = (leptonica_mem != NULL);
	if (!die)
		leptonica_mem = ctx;
	fz_unlock(ctx, FZ_LOCK_ALLOC);
	if (die)
		fz_throw(ctx, FZ_ERROR_GENERIC, "Attempt to use Tesseract from 2 threads at once!");
}

static void
clear_leptonica_mem(fz_context *ctx)
{
	int die = 0;

	fz_lock(ctx, FZ_LOCK_ALLOC);
	die = (leptonica_mem == NULL);
	if (!die)
		leptonica_mem = NULL;
	fz_unlock(ctx, FZ_LOCK_ALLOC);
	if (die)
		fz_throw(ctx, FZ_ERROR_GENERIC, "Attempt to use Tesseract from 2 threads at once!");
}

void *ocr_init(fz_context *ctx, const char *language)
{
	tesseract::TessBaseAPI *api;

	set_leptonica_mem(ctx);
	setPixMemoryManager(my_leptonica_malloc, my_leptonica_free);
	api = new tesseract::TessBaseAPI();

	if (api == NULL)
	{
		clear_leptonica_mem(ctx);
		setPixMemoryManager(malloc, free);
		fz_throw(ctx, FZ_ERROR_GENERIC, "Tesseract initialisation failed");
	}

	if (language == NULL || language[0] == 0)
		language = "eng";

	// Initialize tesseract-ocr with English, without specifying tessdata path
	if (api->Init(NULL, 0, /* data, data_size */
		language,
		tesseract::OcrEngineMode::OEM_DEFAULT,
		NULL, 0, /* configs, configs_size */
		NULL, NULL, /* vars_vec */
		false, /* set_only_non_debug_params */
		&tess_file_reader))
	{
		delete api;
		clear_leptonica_mem(ctx);
		setPixMemoryManager(malloc, free);
		fz_throw(ctx, FZ_ERROR_GENERIC, "Tesseract initialisation failed");
	}

	return api;
}

void ocr_fin(fz_context *ctx, void *api_)
{
	tesseract::TessBaseAPI *api = (tesseract::TessBaseAPI *)api_;

{'fraction_non_alphanumeric': 0.14569767441860465, 'fraction_numerical': 0.0034883720930232558, 'mean_word_length': 4.997907949790795, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6165601', 'n_tokens_mistral': 3579, 'n_tokens_neox': 3434, 'n_words': 824}