<commit_before>//
// Parser is used to invoke the clang libraries to perform actual parsing of
// the input received in the Console.
//
// Part of ccons, the interactive console for the C programming language.
//
// Copyright (c) 2009 <NAME>. This file is distributed under the
// terms of MIT Open Source License. See file LICENSE for details.
//

#include "Parser.h"

#include <iostream>
#include <stack>
#include <algorithm>

#include <llvm/Config/config.h>

#include <clang/AST/AST.h>
#include <clang/AST/ASTConsumer.h>
#include <clang/Basic/FileSystemOptions.h>
#include <clang/Basic/TargetInfo.h>
#include <clang/Basic/TargetOptions.h>
#include <clang/Frontend/FrontendOptions.h>
#include <clang/Frontend/Utils.h>
#include <clang/Lex/HeaderSearch.h>
#include <clang/Lex/LexDiagnostic.h>
#include <clang/Lex/Preprocessor.h>
#include <clang/Parse/ParseAST.h>
#include <clang/Sema/SemaDiagnostic.h>

#include <clang/Basic/TargetInfo.h>
#include <clang/Basic/Diagnostic.h>

#include "Diagnostics.h"
#include "SrcGen.h"

using std::string;

namespace ccons {


//
// ParseOperation
//

ParseOperation::ParseOperation(const clang::LangOptions& options,
                               clang::TargetOptions* targetOptions,
                               clang::DiagnosticsEngine *diag,
                               clang::PPCallbacks *callbacks) :
	_langOpts(options),
	_hsOptions(new clang::HeaderSearchOptions),
	_ppOptions(new clang::PreprocessorOptions),
	_fsOpts(new clang::FileSystemOptions),
	_fm(new clang::FileManager(*_fsOpts)),
	_sm(new clang::SourceManager(*diag, *_fm))
{
	_target.reset(clang::TargetInfo::CreateTargetInfo(*diag, targetOptions));

	_hs.reset(new clang::HeaderSearch(_hsOptions, *_fm, *diag, options, &*_target));
	ApplyHeaderSearchOptions(*_hs, *_hsOptions, options, llvm::Triple(targetOptions->Triple));
	_pp.reset(new clang::Preprocessor(_ppOptions, *diag, _langOpts, &*_target, *_sm, *_hs, *this));
	_pp->addPPCallbacks(callbacks);
	clang::FrontendOptions frontendOptions;
	InitializePreprocessor(*_pp, *_ppOptions, *_hsOptions, frontendOptions);
	_ast.reset(new clang::ASTContext(_langOpts,
	                                 *_sm,
	                                 &*_target,
	                                 _pp->getIdentifierTable(),
	                                 _pp->getSelectorTable(),
	                                 _pp->getBuiltinInfo(),
	                                 0));<commit_msg>apparently TI takes ownership of TO<commit_after>//
// Parser is used to invoke the clang libraries to perform actual parsing of
// the input received in the Console.
//
// Part of ccons, the interactive console for the C programming language.
//
// Copyright (c) 2009 <NAME>. This file is distributed under the
// terms of MIT Open Source License. See file LICENSE for details.
//

#include "Parser.h"

#include <iostream>
#include <stack>
#include <algorithm>

#include <llvm/Config/config.h>

#include <clang/AST/AST.h>
#include <clang/AST/ASTConsumer.h>
#include <clang/Basic/Diagnostic.h>
#include <clang/Basic/FileSystemOptions.h>
#include <clang/Basic/TargetInfo.h>
#include <clang/Basic/TargetOptions.h>
#include <clang/Frontend/FrontendOptions.h>
#include <clang/Frontend/Utils.h>
#include <clang/Lex/HeaderSearch.h>
#include <clang/Lex/LexDiagnostic.h>
#include <clang/Lex/Preprocessor.h>
#include <clang/Parse/ParseAST.h>
#include <clang/Sema/SemaDiagnostic.h>

#include "Diagnostics.h"
#include "SrcGen.h"

using std::string;

namespace ccons {


//
// ParseOperation
//

ParseOperation::ParseOperation(const clang::LangOptions& options,
                               clang::TargetOptions* targetOptions,
                               clang::DiagnosticsEngine *diag,
                               clang::PPCallbacks *callbacks) :
	_langOpts(options),
	_hsOptions(new clang::HeaderSearchOptions),
	_ppOptions(new clang::PreprocessorOptions),
	_fsOpts(new clang::FileSystemOptions),
	_fm(new clang::FileManager(*_fsOpts)),
	_sm(new clang::SourceManager(*diag, *_fm))
{
	_target.reset(clang::TargetInfo::CreateTargetInfo(*diag, new clang::TargetOptions(*targetOptions)));

	_hs.reset(new clang::HeaderSearch(_hsOptions, *_fm, *diag, options, &*_target));
	ApplyHeaderSearchOptions(*_hs, *_hsOptions, options, llvm::Triple(targetOptions->Triple));
	_pp.reset(new clang::Preprocessor(_ppOptions, *diag, _langOpts, &*_target, *_sm, *_hs, *this));
	_pp->addPPCallbacks(callbacks);
	clang::FrontendOptions frontendOptions;
	InitializePreprocessor(*_pp, *_ppOptions, *_hsOptions, frontendOptions);
	_ast.reset(new clang::ASTContext(_langOpts,
	                                 *_sm,
	                                 &*_target,
	                                 _pp->getIdentifierTable(),
	                                 _pp->getSelectorTable(),
	                                 _pp->getBuiltinInfo(),
	                                 0));
{'fraction_non_alphanumeric': 0.12914449447400736, 'fraction_numerical': 0.002046663937781416, 'mean_word_length': 3.2311688311688314, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7672946', 'n_tokens_mistral': 1538, 'n_tokens_neox': 1504, 'n_words': 313}