<commit_before>#include <yuni/yuni.h>
#include <yuni/string.h>
#include <yuni/core/system/main.h>
#include <yuni/io/file.h>
#include <nany/nany.h>
#include <memory>
#include <iostream>
#include <cassert>
#include <limits.h>

using namespace Yuni;

static const char* argv0 = "";



struct Options
{
	//! Verbose mode
	bool verbose = false;
	//! Maximum number of jobs
	uint32_t jobs = 1;
};


static int printUsage(const char* const argv0)
{
	std::cout
		<< "Usage: " << argv0 << " [options] file...\n"
		<< "Options:\n"
		<< "  --bugreport       Display some useful information to report a bug\n"
		<< "                    (https://github.com/nany-lang/nany/issues/new)\n"
		<< "  --help, -h        Display this information\n"
		<< "  --version, -v     Print the version\n\n";
	return EXIT_SUCCESS;
}


static int printBugReportInfo()
{
	auto* text = nany_get_info_for_bugreport();
	if (text)
	{
		std::cout << text;
		::free(text);
		return 0;
	}
	return EXIT_FAILURE;
}


static int printVersion()
{
	std::cout << nany_version() << '\n';
	return EXIT_SUCCESS;
}


static int unknownOption(const AnyString& name)
{
	std::cerr << argv0 << ": unknown option '" << name << "'\n";
	return EXIT_FAILURE;
}



static inline int execute(int argc, char** argv, const Options& options)
{
	assert(argc >= 1);
	String scriptfile;
	IO::Canonicalize(scriptfile, argv[0]);

	// nany context
	nycontext_t ctx;
	nany_initialize(&ctx, nullptr, nullptr);

	// for concurrent build
	if (options.jobs > 1)
		ctx.mt.queueservice = nany_queueservice_create();

	nany_source_add_from_file_n(&ctx, scriptfile.c_str(), scriptfile.size());


	// building
	bool buildstatus;
	{
		nyreport_t* report = nullptr;
		buildstatus = (nany_build(&ctx, &report) == nytrue);

		// report printing
		// if an error has occured or if in verbose mode
		if (YUNI_UNLIKELY((not buildstatus) or options.verbose))
			(buildstatus ? nany_report_print_stdout : nany_report_print_stderr)(report);

		nany_report_unref(&report);
	}


	// executing the program
	int exitstatus = 66;
	if (buildstatus)
	{
		if (argc == 1)
		{
			const char* nargv[] = { scriptfile.c_str(), nullptr };
			exitstatus = nany_run_main(&ctx, 1, nargv);
		}
		else
		{
			auto** nargv = (const char**)::malloc(((size_t) argc + 1) * sizeof(const char**));
			if (nargv)
			{
				for (int i = 1; i < argc; ++i)
					nargv[i] = argv[i];
				nargv[0]    = scriptfile.c_str();
				nargv[argc] = nullptr;

				exitstatus = nany_run_main(&ctx, argc, nargv);
				free(nargv);
			}
		}
	}

	nany_uninitialize(&ctx);
	return exitstatus;
}







YUNI_MAIN_CONSOLE(argc, argv)
{
	argv0 = argv[0];
	if (YUNI_UNLIKELY(argc <= 1))
	{
		std::cerr << argv0 << ": no input script file\n";
		return EXIT_FAILURE;
	}
<commit_msg>added env var NANY_MEMORY_LIMIT (size in bytes)<commit_after>#include <yuni/yuni.h>
#include <yuni/string.h>
#include <yuni/core/system/main.h>
#include <yuni/core/system/environment.h>
#include <yuni/io/file.h>
#include <nany/nany.h>
#include <memory>
#include <iostream>
#include <cassert>
#include <limits.h>

using namespace Yuni;

static const char* argv0 = "";



struct Options
{
	//! Verbose mode
	bool verbose = false;
	//! Maximum number of jobs
	uint32_t jobs = 1;
	//! Memory limit (zero means unlimited)
	size_t memoryLimit = 0;
};


static int printUsage(const char* const argv0)
{
	std::cout
		<< "Usage: " << argv0 << " [options] file...\n"
		<< "Options:\n"
		<< "  --bugreport       Display some useful information to report a bug\n"
		<< "                    (https://github.com/nany-lang/nany/issues/new)\n"
		<< "  --help, -h        Display this information\n"
		<< "  --version, -v     Print the version\n\n";
	return EXIT_SUCCESS;
}


static int printBugReportInfo()
{
	auto* text = nany_get_info_for_bugreport();
	if (text)
	{
		std::cout << text;
		::free(text);
		return 0;
	}
	return EXIT_FAILURE;
}


static int printVersion()
{
	std::cout << nany_version() << '\n';
	return EXIT_SUCCESS;
}


static int unknownOption(const AnyString& name)
{
	std::cerr << argv0 << ": unknown option '" << name << "'\n";
	return EXIT_FAILURE;
}


static inline void initializeContext(nycontext_t& ctx, const Options& options)
{
	size_t memoryLimit = options.memoryLimit;
	if (memoryLimit == 0)
		memoryLimit = static_cast<size_t>(System::Environment::ReadAsUInt64("NANY_MEMORY_LIMIT", 0));

	if (memoryLimit == 0)
	{
		// internal: using nullptr should be slighty faster than
		// using 'nany_memalloc_init_default' + custom memalloc as parameter
		nany_initialize(&ctx, nullptr, nullptr);
	}
	else
	{
		nycontext_memory_t memalloc;
		nany_mem_alloc_init_with_limit(&memalloc, memoryLimit);
		nany_initialize(&ctx, nullptr, &memalloc);
	}

	// for concurrent build, create a queue service if more than
	// one concurrent job is allowed
	if (options.jobs > 1)
		ctx.mt.queueservice = nany_queueservice_create();
}


static inline int execute(int argc, char** argv, const Options& options)
{
	assert(argc >= 1);
	String scriptfile;
	IO::Canonicalize(scriptfile, argv[0]);

	// nany context
	nycontext_t ctx;
	initializeContext(ctx, options);

	// sources
	nany_source_add_from_file_n(&ctx, scriptfile.c_str(), scriptfile.size());

	// building
	bool buildstatus;
	{
		nyreport_t* report = nullptr;
		buildstatus = (nany_build(&ctx, &report) == nytrue);

		// report printing
		// if an error has occured or if in verbose mode
		if (YUNI_UNLIKELY((not buildstatus) or options.verbose))
			(buildstatus ? nany_report_print_stdout : nany_report_print_stderr)(report);

		nany_report_unref(&report);
	}


	// executing the program
	int exitstatus = 66;
	if (buildstatus)
	{
		if (argc == 1)
		{
			const char* nargv[] = { scriptfile.c_str(), nullptr };
			exitstatus = nany_run_main(&ctx, 1, nargv);
		}
		else
		{
			auto** nargv = (const char**)::malloc(((size_t) argc + 1) * sizeof(const char**));
			if (nargv)
			{
				for (int i = 1; i < argc; ++i)
					nargv[i] = argv[i];
				nargv[0]    = scriptfile.c_str();
				nargv[argc] = nullptr;

				exitstatus = nany_run_main(&ctx, argc, nargv);
				free(nargv);
			}
		}
	}

	nany_uninitialize(&ctx);
	return exitstatus;
}






YUNI_MAIN_CONSOLE(argc, argv)
{
	argv0 = argv[0];
	if (YUNI_UNLIKELY(argc <= 1))
	{
		std::cerr << argv0 << ": no input script file\n";
		return EXIT_FAILURE;
	}
{'fraction_non_alphanumeric': 0.149673930332432, 'fraction_numerical': 0.00795291872117067, 'mean_word_length': 3.66815144766147, 'pattern_counts': {'":': 4, '<': 95, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '733966', 'n_tokens_mistral': 2523, 'n_tokens_neox': 2312, 'n_words': 555}