<commit_before>#include "Parser.h"

#include <llvm/Config/config.h>
#include <llvm/Support/MemoryBuffer.h>

#include <clang/AST/ASTConsumer.h>
#include <clang/AST/TranslationUnit.h>
#include <clang/Basic/TargetInfo.h>
#include <clang/Driver/InitHeaderSearch.h>
#include <clang/Lex/HeaderSearch.h>
#include <clang/Lex/Preprocessor.h>
#include <clang/Sema/ParseAST.h>

using std::string;

namespace ccons {

Parser::Parser(const clang::LangOptions& options) :
	_options(options),
	_target(clang::TargetInfo::CreateTargetInfo(LLVM_HOSTTRIPLE))
{
}

void Parser::parse(string src,
                   clang::SourceManager *sm,
                   clang::Diagnostic *diag,
                   clang::ASTConsumer *consumer)
{
	llvm::MemoryBuffer *mb =
		llvm::MemoryBuffer::getMemBufferCopy(&*src.begin(), &*src.end(), "Main");
	assert(mb && "Error creating MemoryBuffer!");
	sm->createMainFileIDForMemBuffer(mb);
	assert(!sm->getMainFileID().isInvalid() && "Error creating MainFileID!");

	clang::HeaderSearch headers(_fm);
	clang::InitHeaderSearch ihs(headers);
	ihs.AddDefaultEnvVarPaths(_options);
	ihs.AddDefaultSystemIncludePaths(_options);
	ihs.Realize();

	clang::Preprocessor *pp =
		new clang::Preprocessor(*diag, _options, *_target, *sm, headers);
	_ast.reset(new clang::ASTContext(_options, *sm, *_target,
		pp->getIdentifierTable(), pp->getSelectorTable()));
	_tu.reset(new clang::TranslationUnit(*_ast));

	clang::ParseAST(*pp, consumer, _ast.get(), _tu.get());
}

} // namespace ccons<commit_msg>update to pass only the TU<commit_after>#include "Parser.h"

#include <llvm/Config/config.h>
#include <llvm/Support/MemoryBuffer.h>

#include <clang/AST/ASTConsumer.h>
#include <clang/AST/TranslationUnit.h>
#include <clang/Basic/TargetInfo.h>
#include <clang/Driver/InitHeaderSearch.h>
#include <clang/Lex/HeaderSearch.h>
#include <clang/Lex/Preprocessor.h>
#include <clang/Sema/ParseAST.h>

using std::string;

namespace ccons {

Parser::Parser(const clang::LangOptions& options) :
	_options(options),
	_target(clang::TargetInfo::CreateTargetInfo(LLVM_HOSTTRIPLE))
{
}

void Parser::parse(string src,
                   clang::SourceManager *sm,
                   clang::Diagnostic *diag,
                   clang::ASTConsumer *consumer)
{
	llvm::MemoryBuffer *mb =
		llvm::MemoryBuffer::getMemBufferCopy(&*src.begin(), &*src.end(), "Main");
	assert(mb && "Error creating MemoryBuffer!");
	sm->createMainFileIDForMemBuffer(mb);
	assert(!sm->getMainFileID().isInvalid() && "Error creating MainFileID!");

	clang::HeaderSearch headers(_fm);
	clang::InitHeaderSearch ihs(headers);
	ihs.AddDefaultEnvVarPaths(_options);
	ihs.AddDefaultSystemIncludePaths(_options);
	ihs.Realize();

	clang::Preprocessor *pp =
		new clang::Preprocessor(*diag, _options, *_target, *sm, headers);
	_ast.reset(new clang::ASTContext(_options, *sm, *_target,
		pp->getIdentifierTable(), pp->getSelectorTable()));
	_tu.reset(new clang::TranslationUnit(*_ast));

	clang::ParseAST(*pp, consumer, _tu.get());
}

} // namespace ccons
{'fraction_non_alphanumeric': 0.1617304492512479, 'fraction_numerical': 0.0, 'mean_word_length': 5.650442477876106, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6359222', 'n_tokens_mistral': 1040, 'n_tokens_neox': 1011, 'n_words': 165}