<commit_before>/*
 * Copyright (C) 2016,2017 deipi.com LLC and contributors. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#pragma once

#include "xapiand.h"


#if XAPIAND_V8

#include <array>               // for array
#include <atomic>              // for atomic_bool
#include <condition_variable>  // for condition_variable
#include <mutex>               // for mutex
#include <stdio.h>             // for size_t, snprintf
#include <stdlib.h>            // for malloc, free
#include <string.h>            // for memset
#include <string>              // for string
#include <thread>              // for thread
#include <unordered_map>       // for unordered_map

#include "lru.h"               // for LRU
#include "wrapper.h"           // for wrap


namespace v8pp {

// v8 version supported: v8-5.1


constexpr auto DURATION_SCRIPT = std::chrono::milliseconds(100);


inline static size_t hash(const std::string& source) {
	std::hash<std::string> hash_fn;
	return hash_fn(source);
}


static std::string report_exception(v8::TryCatch* try_catch) {
	auto exception_string = convert<std::string>()(try_catch->Exception());

	v8::Local<v8::Message> message = try_catch->Message();
	if (message.IsEmpty()) {
		// V8 didn't provide any extra information about this error;
		// just print the exception.
		fprintf(stderr, "%s\n", exception_string.c_str());
		return exception_string;
	} else {
		// Print (filename):(line number): (message).
		fprintf(stderr, "%s:%i: %s\n", convert<const char*>()(v8::String::Utf8Value(message->GetScriptResourceName())), message->GetLineNumber(), exception_string.c_str());

		// Print line of source code.
		fprintf(stderr, "%s\n", convert<const char*>()(v8::String::Utf8Value(message->GetSourceLine())));

		// Print wavy underline (GetUnderline is deprecated).
		int start = message->GetStartColumn();
		for (int i = 0; i < start; ++i) {
			fprintf(stderr, " ");
		}
		int end = message->GetEndColumn();
		for (int i = start; i < end; ++i) {
			fprintf(stderr, "^");
		}
		fprintf(stderr, "\n");
		return exception_string;
	}
}


static void print(const v8::FunctionCallbackInfo<v8::Value>& args) {
	auto argc = args.Length();
	if (argc) {
		auto convertor = convert<MsgPack>();
		printf("%s ", convert<const char*>()(v8::String::Utf8Value(args[0])));
		for (int i = 1; i < argc; ++i) {
			printf(" %s", convertor(args[i]).to_string().c_str());
		}
	}
	printf("\n");
}


static void return_data(const v8::FunctionCallbackInfo<v8::Value>& args) {
	args.GetReturnValue().Set(args.Data());
}


class Processor {

	class ArrayBufferAllocator : public v8::ArrayBuffer::Allocator {
	public:
		void* AllocateUninitialized(size_t length) {
			void* data = malloc(length);
			return data;
		}

		void* Allocate(size_t length) {
			void* data = AllocateUninitialized(length);
			return data == nullptr ? data : memset(data, 0, length);
		}

		void Free(void* data, size_t) {
			free(data);
		}
	};


	class ScriptLRU : public lru::LRU<size_t, std::pair<size_t, std::shared_ptr<Processor>>> {
	public:
		ScriptLRU(ssize_t max_size)
			: LRU(max_size) { }
	};
<commit_msg>Declaring explicit constructor in v8pp::ScriptLRU<commit_after>/*
 * Copyright (C) 2016,2017 deipi.com LLC and contributors. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#pragma once

#include "xapiand.h"


#if XAPIAND_V8

#include <array>               // for array
#include <atomic>              // for atomic_bool
#include <condition_variable>  // for condition_variable
#include <mutex>               // for mutex
#include <stdio.h>             // for size_t, snprintf
#include <stdlib.h>            // for malloc, free
#include <string.h>            // for memset
#include <string>              // for string
#include <thread>              // for thread
#include <unordered_map>       // for unordered_map

#include "lru.h"               // for LRU
#include "wrapper.h"           // for wrap


namespace v8pp {

// v8 version supported: v8-5.1


constexpr auto DURATION_SCRIPT = std::chrono::milliseconds(100);


inline static size_t hash(const std::string& source) {
	std::hash<std::string> hash_fn;
	return hash_fn(source);
}


static std::string report_exception(v8::TryCatch* try_catch) {
	auto exception_string = convert<std::string>()(try_catch->Exception());

	v8::Local<v8::Message> message = try_catch->Message();
	if (message.IsEmpty()) {
		// V8 didn't provide any extra information about this error;
		// just print the exception.
		fprintf(stderr, "%s\n", exception_string.c_str());
		return exception_string;
	} else {
		// Print (filename):(line number): (message).
		fprintf(stderr, "%s:%i: %s\n", convert<const char*>()(v8::String::Utf8Value(message->GetScriptResourceName())), message->GetLineNumber(), exception_string.c_str());

		// Print line of source code.
		fprintf(stderr, "%s\n", convert<const char*>()(v8::String::Utf8Value(message->GetSourceLine())));

		// Print wavy underline (GetUnderline is deprecated).
		int start = message->GetStartColumn();
		for (int i = 0; i < start; ++i) {
			fprintf(stderr, " ");
		}
		int end = message->GetEndColumn();
		for (int i = start; i < end; ++i) {
			fprintf(stderr, "^");
		}
		fprintf(stderr, "\n");
		return exception_string;
	}
}


static void print(const v8::FunctionCallbackInfo<v8::Value>& args) {
	auto argc = args.Length();
	if (argc) {
		auto convertor = convert<MsgPack>();
		printf("%s ", convert<const char*>()(v8::String::Utf8Value(args[0])));
		for (int i = 1; i < argc; ++i) {
			printf(" %s", convertor(args[i]).to_string().c_str());
		}
	}
	printf("\n");
}


static void return_data(const v8::FunctionCallbackInfo<v8::Value>& args) {
	args.GetReturnValue().Set(args.Data());
}


class Processor {

	class ArrayBufferAllocator : public v8::ArrayBuffer::Allocator {
	public:
		void* AllocateUninitialized(size_t length) {
			void* data = malloc(length);
			return data;
		}

		void* Allocate(size_t length) {
			void* data = AllocateUninitialized(length);
			return data == nullptr ? data : memset(data, 0, length);
		}

		void Free(void* data, size_t) {
			free(data);
		}
	};


	class ScriptLRU : public lru::LRU<size_t, std::pair<size_t, std::shared_ptr<Processor>>> {
	public:
		explicit ScriptLRU(ssize_t max_size)
			: LRU(max_size) { }
	};

{'fraction_non_alphanumeric': 0.13133612941881367, 'fraction_numerical': 0.008747753145596166, 'mean_word_length': 3.807603686635945, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2526284', 'n_tokens_mistral': 2745, 'n_tokens_neox': 2555, 'n_words': 821}