<commit_before>/*
	This file is part of cpp-ethereum.

	cpp-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	cpp-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/**
 * @author Christian <<EMAIL>>
 * @date 2014
 * Solidity compiler.
 */

#include <libsolidity/Compiler.h>
#include <algorithm>
#include <boost/range/adaptor/reversed.hpp>
#include <libevmcore/Instruction.h>
#include <libevmasm/Assembly.h>
#include <libsolidity/AST.h>
#include <libsolidity/ExpressionCompiler.h>
#include <libsolidity/CompilerUtils.h>

using namespace std;

namespace dev {
namespace solidity {

/**
 * Simple helper class to ensure that the stack height is the same at certain places in the code.
 */
class StackHeightChecker
{
public:
	StackHeightChecker(CompilerContext const& _context):
		m_context(_context), stackHeight(m_context.getStackHeight()) {}
	void check() { solAssert(m_context.getStackHeight() == stackHeight, "I sense a disturbance in the stack."); }
private:
	CompilerContext const& m_context;
	unsigned stackHeight;
};

void Compiler::compileContract(ContractDefinition const& _contract,
							   map<ContractDefinition const*, bytes const*> const& _contracts)
{
	m_context = CompilerContext(); // clear it just in case
	{
		CompilerContext::LocationSetter locationSetterRunTime(m_context, _contract);
		CompilerUtils(m_context).initialiseFreeMemoryPointer();
		initializeContext(_contract, _contracts);
		appendFunctionSelector(_contract);
		set<Declaration const*> functions = m_context.getFunctionsWithoutCode();
		while (!functions.empty())
		{
			for (Declaration const* function: functions)
			{
				m_context.setStackOffset(0);
				function->accept(*this);
			}
			functions = m_context.getFunctionsWithoutCode();
		}
	}

	// Swap the runtime context with the creation-time context
	swap(m_context, m_runtimeContext);
	CompilerContext::LocationSetter locationSetterCreationTime(m_context, _contract);
	CompilerUtils(m_context).initialiseFreeMemoryPointer();
	initializeContext(_contract, _contracts);
	packIntoContractCreator(_contract, m_runtimeContext);
	if (m_optimize)
		m_context.optimise(m_optimizeRuns);
}

eth::AssemblyItem Compiler::getFunctionEntryLabel(FunctionDefinition const& _function) const
{
	return m_runtimeContext.getFunctionEntryLabelIfExists(_function);
}

void Compiler::initializeContext(ContractDefinition const& _contract,
								 map<ContractDefinition const*, bytes const*> const& _contracts)
{
	m_context.setCompiledContracts(_contracts);
	m_context.setInheritanceHierarchy(_contract.getLinearizedBaseContracts());
	registerStateVariables(_contract);
	m_context.resetVisitedNodes(&_contract);
}

void Compiler::packIntoContractCreator(ContractDefinition const& _contract, CompilerContext const& _runtimeContext)
{
	// Determine the arguments that are used for the base constructors.
	std::vector<ContractDefinition const*> const& bases = _contract.getLinearizedBaseContracts();
	for (ContractDefinition const* contract: bases)
	{
		if (FunctionDefinition const* constructor = contract->getConstructor())
			for (auto const& modifier: constructor->getModifiers())
			{
				auto baseContract = dynamic_cast<ContractDefinition const*>(
					&modifier->getName()->getReferencedDeclaration());
				if (baseContract)
					if (m_baseArguments.count(baseContract->getConstructor()) == 0)
						m_baseArguments[baseContract->getConstructor()] = &modifier->getArguments();
			}

		for (ASTPointer<InheritanceSpecifier> const& base: contract->getBaseContracts())
		{
			ContractDefinition const* baseContract = dynamic_cast<ContractDefinition const*>(
						&base->getName()->getReferencedDeclaration());
			solAssert(baseContract, "");

			if (m_baseArguments.count(baseContract->getConstructor()) == 0)
				m_baseArguments[baseContract->getConstructor()] = &base->getArguments();
		}
	}
	// Initialization of state variables in base-to-derived order.
	for (ContractDefinition const* contract: boost::adaptors::reverse(bases))
		initializeStateVariables(*contract);

	if (FunctionDefinition const* constructor = _contract.getConstructor())
		appendConstructor(*constructor);
	else if (auto c = m_context.getNextConstructor(_contract))
		appendBaseConstructor(*c);

	eth::AssemblyItem runtimeSub = m_context.addSubroutine(_runtimeContext.getAssembly());
	solAssert(runtimeSub.data() < numeric_limits<size_t>::max(), "");
	m_runtimeSub = size_t(runtimeSub.data());
	// stack contains sub size
	m_context << eth::Instruction::DUP1 << runtimeSub << u256(0) << eth::Instruction::CODECOPY;
	m_context << u256(0) << eth::Instruction::RETURN;

	// note that we have to include the functions again because of absolute jump labels
	set<Declaration const*> functions = m_context.getFunctionsWithoutCode();
	while (!functions.empty())
	{
		for (Declaration const* function: functions)
			function->accept(*this);
		functions = m_context.getFunctionsWithoutCode();
	}
}

void Compiler::appendBaseConstructor(FunctionDefinition const& _constructor)
{
	CompilerContext::LocationSetter locationSetter(m_context, _constructor);
	FunctionType constructorType(_constructor);
	if (!constructorType.getParameterTypes().empty())
	{
		solAssert(m_baseArguments.count(&_constructor), "");
		std::vector<ASTPointer<Expression>> const* arguments = m_baseArguments[&_constructor];
		solAssert(arguments, "");
		for (unsigned i = 0; i < arguments->size(); ++i)
			compileExpression(*(arguments->at(i)), constructorType.getParameterTypes()[i]);
	}
	_constructor.accept(*this);
}

void Compiler::appendConstructor(FunctionDefinition const& _constructor)
{
	CompilerContext::LocationSetter locationSetter(m_context, _constructor);
	// copy constructor arguments from code to memory and then to stack, they are supplied after the actual program
	unsigned argumentSize = 0;
	for (ASTPointer<VariableDeclaration> const& var: _constructor.getParameters())
		argumentSize += var->getType()->getCalldataEncodedSize();

	if (argumentSize > 0)
	{
		m_context << u256(argumentSize);
		m_context.appendProgramSize();
		m_context << u256(CompilerUtils::dataStartOffset); // copy it to byte four as expected for ABI calls
		m_context << eth::Instruction::CODECOPY;
		appendCalldataUnpacker(FunctionType(_constructor).getParameterTypes(), true);
	}
	_constructor.accept(*this);
}

void Compiler::appendFunctionSelector(ContractDefinition const& _contract)
{
	map<FixedHash<4>, FunctionTypePointer> interfaceFunctions = _contract.getInterfaceFunctions();
	map<FixedHash<4>, const eth::AssemblyItem> callDataUnpackerEntryPoints;

	FunctionDefinition const* fallback = _contract.getFallbackFunction();
	eth::AssemblyItem notFound = m_context.newTag();
	// shortcut messages without data if we have many functions in order to be able to receive
	// ether with constant gas
	if (interfaceFunctions.size() > 5 || fallback)
	{
		m_context << eth::Instruction::CALLDATASIZE << eth::Instruction::ISZERO;
		m_context.appendConditionalJumpTo(notFound);
	}

	// retrieve the function signature hash from the calldata
	if (!interfaceFunctions.empty())
		CompilerUtils(m_context).loadFromMemory(0, IntegerType(CompilerUtils::dataStartOffset * 8), true);

	// stack now is: 1 0 <funhash>
	for (auto const& it: interfaceFunctions)
	{
		callDataUnpackerEntryPoints.insert(std::make_pair(it.first, m_context.newTag()));
		m_context << eth::dupInstruction(1) << u256(FixedHash<4>::Arith(it.first)) << eth::Instruction::EQ;
		m_context.appendConditionalJumpTo(callDataUnpackerEntryPoints.at(it.first));
	}
	m_context.appendJumpTo(notFound);

	m_context << notFound;
	if (fallback)
	{
		eth::AssemblyItem returnTag = m_context.pushNewTag();
		fallback->accept(*this);
		m_context << returnTag;
		appendReturnValuePacker(FunctionType(*fallback).getReturnParameterTypes());
	}
	else
		m_context << eth::Instruction::STOP; // function not found

	for (auto const& it: interfaceFunctions)
	{
		FunctionTypePointer const& functionType = it.second;
		solAssert(functionType->hasDeclaration(), "");
		CompilerContext::LocationSetter locationSetter(m_context, functionType->getDeclaration());
		m_context << callDataUnpackerEntryPoints.at(it.first);
		eth::AssemblyItem returnTag = m_context.pushNewTag();
		appendCalldataUnpacker(functionType->getParameterTypes());
		m_context.appendJumpTo(m_context.getFunctionEntryLabel(functionType->getDeclaration()));
		m_context << returnTag;
		appendReturnValuePacker(functionType->getReturnParameterTypes());
	}
}

void Compiler::appendCalldataUnpacker(TypePointers const& _typeParameters, bool _fromMemory)
{
	// We do not check the calldata size, everything is zero-paddedd

	m_context << u256(CompilerUtils::dataStartOffset);
	for (TypePointer const& type: _typeParameters)
	{
		if (type->getCategory() == Type::Category::Array)
		{
			auto const& arrayType = dynamic_cast<ArrayType const&>(*type);
			if (arrayType.location() == ReferenceType::Location::CallData)
			{
				if (type->isDynamicallySized())
				{
					// put on stack: data_pointer length
					CompilerUtils(m_context).loadFromMemoryDynamic(IntegerType(256), !_fromMemory);
					// stack: data_offset next_pointer
					//@todo once we support nested arrays, this offset needs to be dynamic.
					m_context << eth::Instruction::SWAP1 << u256(CompilerUtils::dataStartOffset);
					m_context << eth::Instruction::ADD;
					// stack: next_pointer data_pointer
					// retrieve length
					CompilerUtils(m_context).loadFromMemoryDynamic(IntegerType(256), !_fromMemory, true);
					// stack: next_pointer length data_pointer
					m_context << eth::Instruction::SWAP2;
				}
				else
				{
					// leave the pointer on the stack
					m_context << eth::Instruction::DUP1;
					m_context << u256(type->getCalldataEncodedSize()) << eth::Instruction::ADD;
				}
			}
			else
			{
				solAssert(arrayType.location() == ReferenceType::Location::Memory, "");
				CompilerUtils(m_context).fetchFreeMemoryPointer();
				CompilerUtils(m_context).storeInMemoryDynamic(*type);
				CompilerUtils(m_context).storeFreeMemoryPointer();
			}
		}
		else
		{
			solAssert(!type->isDynamicallySized(), "Unknown dynamically sized type: " + type->toString());
			CompilerUtils(m_context).loadFromMemoryDynamic(*type, !_fromMemory, true);
		}
	}
	m_context << eth::Instruction::POP;
}

void Compiler::appendReturnValuePacker(TypePointers const& _typeParameters)
{
	unsigned dataOffset = 0;
	unsigned stackDepth = 0;
	for (TypePointer const& type: _typeParameters)
		stackDepth += type->getSizeOnStack();

	for (TypePointer const& type: _typeParameters)
	{
		CompilerUtils(m_context).copyToStackTop(stackDepth, type->getSizeOnStack());
		ExpressionCompiler(m_context, m_optimize).appendTypeConversion(*type, *type, true);
		bool const c_padToWords = true;
		dataOffset += CompilerUtils(m_context).storeInMemory(dataOffset, *type, c_padToWords);
		stackDepth -= type->getSizeOnStack();
	}
	// note that the stack is not cleaned up here
	m_context << u256(dataOffset) << u256(0) << eth::Instruction::RETURN;
}

void Compiler::registerStateVariables(ContractDefinition const& _contract)
{
	for (auto const& var: ContractType(_contract).getStateVariables())
		m_context.addStateVariable(*get<0>(var), get<1>(var), get<2>(var));
}<commit_msg>Optimize RETURN x 0 to STOP.<commit_after>/*
	This file is part of cpp-ethereum.

	cpp-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	cpp-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/**
 * @author Christian <<EMAIL>>
 * @date 2014
 * Solidity compiler.
 */

#include <libsolidity/Compiler.h>
#include <algorithm>
#include <boost/range/adaptor/reversed.hpp>
#include <libevmcore/Instruction.h>
#include <libevmasm/Assembly.h>
#include <libsolidity/AST.h>
#include <libsolidity/ExpressionCompiler.h>
#include <libsolidity/CompilerUtils.h>

using namespace std;

namespace dev {
namespace solidity {

/**
 * Simple helper class to ensure that the stack height is the same at certain places in the code.
 */
class StackHeightChecker
{
public:
	StackHeightChecker(CompilerContext const& _context):
		m_context(_context), stackHeight(m_context.getStackHeight()) {}
	void check() { solAssert(m_context.getStackHeight() == stackHeight, "I sense a disturbance in the stack."); }
private:
	CompilerContext const& m_context;
	unsigned stackHeight;
};

void Compiler::compileContract(ContractDefinition const& _contract,
							   map<ContractDefinition const*, bytes const*> const& _contracts)
{
	m_context = CompilerContext(); // clear it just in case
	{
		CompilerContext::LocationSetter locationSetterRunTime(m_context, _contract);
		CompilerUtils(m_context).initialiseFreeMemoryPointer();
		initializeContext(_contract, _contracts);
		appendFunctionSelector(_contract);
		set<Declaration const*> functions = m_context.getFunctionsWithoutCode();
		while (!functions.empty())
		{
			for (Declaration const* function: functions)
			{
				m_context.setStackOffset(0);
				function->accept(*this);
			}
			functions = m_context.getFunctionsWithoutCode();
		}
	}

	// Swap the runtime context with the creation-time context
	swap(m_context, m_runtimeContext);
	CompilerContext::LocationSetter locationSetterCreationTime(m_context, _contract);
	CompilerUtils(m_context).initialiseFreeMemoryPointer();
	initializeContext(_contract, _contracts);
	packIntoContractCreator(_contract, m_runtimeContext);
	if (m_optimize)
		m_context.optimise(m_optimizeRuns);
}

eth::AssemblyItem Compiler::getFunctionEntryLabel(FunctionDefinition const& _function) const
{
	return m_runtimeContext.getFunctionEntryLabelIfExists(_function);
}

void Compiler::initializeContext(ContractDefinition const& _contract,
								 map<ContractDefinition const*, bytes const*> const& _contracts)
{
	m_context.setCompiledContracts(_contracts);
	m_context.setInheritanceHierarchy(_contract.getLinearizedBaseContracts());
	registerStateVariables(_contract);
	m_context.resetVisitedNodes(&_contract);
}

void Compiler::packIntoContractCreator(ContractDefinition const& _contract, CompilerContext const& _runtimeContext)
{
	// Determine the arguments that are used for the base constructors.
	std::vector<ContractDefinition const*> const& bases = _contract.getLinearizedBaseContracts();
	for (ContractDefinition const* contract: bases)
	{
		if (FunctionDefinition const* constructor = contract->getConstructor())
			for (auto const& modifier: constructor->getModifiers())
			{
				auto baseContract = dynamic_cast<ContractDefinition const*>(
					&modifier->getName()->getReferencedDeclaration());
				if (baseContract)
					if (m_baseArguments.count(baseContract->getConstructor()) == 0)
						m_baseArguments[baseContract->getConstructor()] = &modifier->getArguments();
			}

		for (ASTPointer<InheritanceSpecifier> const& base: contract->getBaseContracts())
		{
			ContractDefinition const* baseContract = dynamic_cast<ContractDefinition const*>(
						&base->getName()->getReferencedDeclaration());
			solAssert(baseContract, "");

			if (m_baseArguments.count(baseContract->getConstructor()) == 0)
				m_baseArguments[baseContract->getConstructor()] = &base->getArguments();
		}
	}
	// Initialization of state variables in base-to-derived order.
	for (ContractDefinition const* contract: boost::adaptors::reverse(bases))
		initializeStateVariables(*contract);

	if (FunctionDefinition const* constructor = _contract.getConstructor())
		appendConstructor(*constructor);
	else if (auto c = m_context.getNextConstructor(_contract))
		appendBaseConstructor(*c);

	eth::AssemblyItem runtimeSub = m_context.addSubroutine(_runtimeContext.getAssembly());
	solAssert(runtimeSub.data() < numeric_limits<size_t>::max(), "");
	m_runtimeSub = size_t(runtimeSub.data());
	// stack contains sub size
	m_context << eth::Instruction::DUP1 << runtimeSub << u256(0) << eth::Instruction::CODECOPY;
	m_context << u256(0) << eth::Instruction::RETURN;

	// note that we have to include the functions again because of absolute jump labels
	set<Declaration const*> functions = m_context.getFunctionsWithoutCode();
	while (!functions.empty())
	{
		for (Declaration const* function: functions)
			function->accept(*this);
		functions = m_context.getFunctionsWithoutCode();
	}
}

void Compiler::appendBaseConstructor(FunctionDefinition const& _constructor)
{
	CompilerContext::LocationSetter locationSetter(m_context, _constructor);
	FunctionType constructorType(_constructor);
	if (!constructorType.getParameterTypes().empty())
	{
		solAssert(m_baseArguments.count(&_constructor), "");
		std::vector<ASTPointer<Expression>> const* arguments = m_baseArguments[&_constructor];
		solAssert(arguments, "");
		for (unsigned i = 0; i < arguments->size(); ++i)
			compileExpression(*(arguments->at(i)), constructorType.getParameterTypes()[i]);
	}
	_constructor.accept(*this);
}

void Compiler::appendConstructor(FunctionDefinition const& _constructor)
{
	CompilerContext::LocationSetter locationSetter(m_context, _constructor);
	// copy constructor arguments from code to memory and then to stack, they are supplied after the actual program
	unsigned argumentSize = 0;
	for (ASTPointer<VariableDeclaration> const& var: _constructor.getParameters())
		argumentSize += var->getType()->getCalldataEncodedSize();

	if (argumentSize > 0)
	{
		m_context << u256(argumentSize);
		m_context.appendProgramSize();
		m_context << u256(CompilerUtils::dataStartOffset); // copy it to byte four as expected for ABI calls
		m_context << eth::Instruction::CODECOPY;
		appendCalldataUnpacker(FunctionType(_constructor).getParameterTypes(), true);
	}
	_constructor.accept(*this);
}

void Compiler::appendFunctionSelector(ContractDefinition const& _contract)
{
	map<FixedHash<4>, FunctionTypePointer> interfaceFunctions = _contract.getInterfaceFunctions();
	map<FixedHash<4>, const eth::AssemblyItem> callDataUnpackerEntryPoints;

	FunctionDefinition const* fallback = _contract.getFallbackFunction();
	eth::AssemblyItem notFound = m_context.newTag();
	// shortcut messages without data if we have many functions in order to be able to receive
	// ether with constant gas
	if (interfaceFunctions.size() > 5 || fallback)
	{
		m_context << eth::Instruction::CALLDATASIZE << eth::Instruction::ISZERO;
		m_context.appendConditionalJumpTo(notFound);
	}

	// retrieve the function signature hash from the calldata
	if (!interfaceFunctions.empty())
		CompilerUtils(m_context).loadFromMemory(0, IntegerType(CompilerUtils::dataStartOffset * 8), true);

	// stack now is: 1 0 <funhash>
	for (auto const& it: interfaceFunctions)
	{
		callDataUnpackerEntryPoints.insert(std::make_pair(it.first, m_context.newTag()));
		m_context << eth::dupInstruction(1) << u256(FixedHash<4>::Arith(it.first)) << eth::Instruction::EQ;
		m_context.appendConditionalJumpTo(callDataUnpackerEntryPoints.at(it.first));
	}
	m_context.appendJumpTo(notFound);

	m_context << notFound;
	if (fallback)
	{
		eth::AssemblyItem returnTag = m_context.pushNewTag();
		fallback->accept(*this);
		m_context << returnTag;
		appendReturnValuePacker(FunctionType(*fallback).getReturnParameterTypes());
	}
	else
		m_context << eth::Instruction::STOP; // function not found

	for (auto const& it: interfaceFunctions)
	{
		FunctionTypePointer const& functionType = it.second;
		solAssert(functionType->hasDeclaration(), "");
		CompilerContext::LocationSetter locationSetter(m_context, functionType->getDeclaration());
		m_context << callDataUnpackerEntryPoints.at(it.first);
		eth::AssemblyItem returnTag = m_context.pushNewTag();
		appendCalldataUnpacker(functionType->getParameterTypes());
		m_context.appendJumpTo(m_context.getFunctionEntryLabel(functionType->getDeclaration()));
		m_context << returnTag;
		appendReturnValuePacker(functionType->getReturnParameterTypes());
	}
}

void Compiler::appendCalldataUnpacker(TypePointers const& _typeParameters, bool _fromMemory)
{
	// We do not check the calldata size, everything is zero-paddedd

	m_context << u256(CompilerUtils::dataStartOffset);
	for (TypePointer const& type: _typeParameters)
	{
		if (type->getCategory() == Type::Category::Array)
		{
			auto const& arrayType = dynamic_cast<ArrayType const&>(*type);
			if (arrayType.location() == ReferenceType::Location::CallData)
			{
				if (type->isDynamicallySized())
				{
					// put on stack: data_pointer length
					CompilerUtils(m_context).loadFromMemoryDynamic(IntegerType(256), !_fromMemory);
					// stack: data_offset next_pointer
					//@todo once we support nested arrays, this offset needs to be dynamic.
					m_context << eth::Instruction::SWAP1 << u256(CompilerUtils::dataStartOffset);
					m_context << eth::Instruction::ADD;
					// stack: next_pointer data_pointer
					// retrieve length
					CompilerUtils(m_context).loadFromMemoryDynamic(IntegerType(256), !_fromMemory, true);
					// stack: next_pointer length data_pointer
					m_context << eth::Instruction::SWAP2;
				}
				else
				{
					// leave the pointer on the stack
					m_context << eth::Instruction::DUP1;
					m_context << u256(type->getCalldataEncodedSize()) << eth::Instruction::ADD;
				}
			}
			else
			{
				solAssert(arrayType.location() == ReferenceType::Location::Memory, "");
				CompilerUtils(m_context).fetchFreeMemoryPointer();
				CompilerUtils(m_context).storeInMemoryDynamic(*type);
				CompilerUtils(m_context).storeFreeMemoryPointer();
			}
		}
		else
		{
			solAssert(!type->isDynamicallySized(), "Unknown dynamically sized type: " + type->toString());
			CompilerUtils(m_context).loadFromMemoryDynamic(*type, !_fromMemory, true);
		}
	}
	m_context << eth::Instruction::POP;
}

void Compiler::appendReturnValuePacker(TypePointers const& _typeParameters)
{
	unsigned dataOffset = 0;
	unsigned stackDepth = 0;
	for (TypePointer const& type: _typeParameters)
		stackDepth += type->getSizeOnStack();

	for (TypePointer const& type: _typeParameters)
	{
		CompilerUtils(m_context).copyToStackTop(stackDepth, type->getSizeOnStack());
		ExpressionCompiler(m_context, m_optimize).appendTypeConversion(*type, *type, true);
		bool const c_padToWords = true;
		dataOffset += CompilerUtils(m_context).storeInMemory(dataOffset, *type, c_padToWords);
		stackDepth -= type->getSizeOnStack();
	}
	// note that the stack is not cleaned up here
	if (dataOffset == 0)
		m_context << eth::Instruction::STOP;
	else
		m_context << u256(dataOffset) << u256(0) << eth::Instruction::RETURN;
}

void Compiler::registerStateVariables(ContractDefinition const& _contract)
{
	for (auto const& var: ContractType(_contract).getStateVariables())
		m_context.addStateVariable(*get<0>(var), get<1>(var), get<2>(var));
}
{'fraction_non_alphanumeric': 0.11728105689324526, 'fraction_numerical': 0.005881099509908374, 'mean_word_length': 6.0723327305605785, 'pattern_counts': {'":': 0, '<': 199, '<?xml version=': 0, '>': 139, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7337041', 'n_tokens_mistral': 7366, 'n_tokens_neox': 6754, 'n_words': 1551}