<commit_before>/*
 * Main.cpp
 *
 *  Created on: 6.7.2012
 *      Author: ondra
 */

#include "test.h"


#include "tempe/compiler.h"
#include "lightspeed/base/streams/standardIO.tcc"
#include "tempe/varTable.h"
#include "tempe/commentRemover.h"
#include "lightspeed/base/iter/iteratorFilter.tcc"
#include <lightspeed/utils/json/jsonfast.tcc>

#include "tempe/tempeCompiler.h"
namespace TempeTest {

using namespace Tempe;

static Main theApp;


integer Main::start(const Args& args) {

	CommentRemover commentless(StdInput().getHandle().get());commentless.setStaticObj();
	ConsoleA console(&commentless,
			ConsoleA::getStdOutput(),ConsoleA::getStdError());


	VarTable vtable;
	//console.scan.setWS(ConstStrA(" \t\n\r\a\b"));
	while (console.scan.hasItems()) {

		TempeCompiler comp(TempeCompiler::ctnPlain);
		PExprNode nd;
		try {
			console.scan("\b%");
			 nd = comp.compile(console.scan);
		} catch (const std::exception &e) {
			console.error("Compile error: %1\n") << e.what();
			console.scan("%(*)0\n%");
			console.flush();
			continue;
		}

		try {
			Value k = nd->calculate(vtable);<commit_msg>getHandle -> getStream<commit_after>/*
 * Main.cpp
 *
 *  Created on: 6.7.2012
 *      Author: ondra
 */

#include "test.h"


#include "tempe/compiler.h"
#include "lightspeed/base/streams/standardIO.tcc"
#include "tempe/varTable.h"
#include "tempe/commentRemover.h"
#include "lightspeed/base/iter/iteratorFilter.tcc"
#include <lightspeed/utils/json/jsonfast.tcc>

#include "tempe/tempeCompiler.h"
namespace TempeTest {

using namespace Tempe;

static Main theApp;


integer Main::start(const Args& args) {

	CommentRemover commentless(StdInput().getStream());commentless.setStaticObj();
	ConsoleA console(&commentless,
			ConsoleA::getStdOutput(),ConsoleA::getStdError());


	VarTable vtable;
	//console.scan.setWS(ConstStrA(" \t\n\r\a\b"));
	while (console.scan.hasItems()) {

		TempeCompiler comp(TempeCompiler::ctnPlain);
		PExprNode nd;
		try {
			console.scan("\b%");
			 nd = comp.compile(console.scan);
		} catch (const std::exception &e) {
			console.error("Compile error: %1\n") << e.what();
			console.scan("%(*)0\n%");
			console.flush();
			continue;
		}

		try {
			Value k = nd->calculate(vtable);
{'fraction_non_alphanumeric': 0.170797656602073, 'fraction_numerical': 0.007210455159981974, 'mean_word_length': 4.481481481481482, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2524346', 'n_tokens_mistral': 869, 'n_tokens_neox': 794, 'n_words': 138}