<commit_before>/*
** SelfPortrait API
** See Copyright Notice in reflection.h
*/
#include "llvm/Support/Host.h"

#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/Tool.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"

#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/Utils.h"
#include "clang/Basic/TargetOptions.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/ASTConsumer.h"
#include "clang/Frontend/ASTUnit.h"
#include "clang/Parse/Parser.h"
#include "clang/Parse/ParseAST.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclTemplate.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Sema/Sema.h"
#include "clang/Sema/Template.h"


#include <vector>
#include <string>
#include <set>
#include <iostream>
#include <fstream>
#include <sstream>
#include <stdexcept>
#include <unordered_map>

#include "definitions.h"
using namespace definitions;

using namespace std;
using namespace clang;
using namespace llvm;

template<class... Args, template<class...> class Associative>
bool contains(const Associative<Args...>& container, const typename Associative<Args...>::key_type& key)
{
	auto it = container.find(key);
	return (it != container.end());
}

template<class... Args, template<class...> class Container>
std::string join(const Container<Args...>& container, const std::string& separator)
{
	if (container.empty()) {
		return "";
	} else {

		auto it = container.begin();

		std::string ret = *it;

		++it;
		for (; it != container.end(); ++it) {
			ret += separator + *it;
		}
		return ret;
	}
}


class MyASTConsumer
{
	TranslationUnit m_tu;
	TranslationUnitBuilder m_builder;

	SourceManager* m_sourceManager;
	PrintingPolicy m_printPol;
	set<Decl*> m_visited;
	Sema& m_sema;
	bool m_instantiateTemplates;


	struct IncompleteType {
		QualType type;
		SourceRange range;
	};

	struct PrivateType {};

	string printType(string orig) {
		string ret;

		for (auto c: orig) {
			if (c == ',') {
				ret += " COMMA ";
			} else {
				ret += c;
			}
		}
		return ret;
	}

	string printType(QualType t) {
		return printType(t.getAsString(m_printPol));
	}

	void treatIncompleteType (IncompleteType t, Decl* decl) {
		SourceLocation start = t.range.getBegin();
		const FileEntry* entry = m_sourceManager->getFileEntryForID(m_sourceManager->getFileID(start));
		int line = m_sourceManager->getSpellingLineNumber(start);
		int column = m_sourceManager->getSpellingColumnNumber(start);



		cerr << entry->getName() << ":" <<
				line << ":" << column <<
				": warning: Incomplete type " <<
				t.type.getAsString(m_printPol) <<

				",\nmeta data will not be emitted for ";

		if (NamedDecl* nd = dyn_cast<NamedDecl>(decl)) {
			cerr << nd->getNameAsString() << endl;
		} else {
			cerr << "the current declaration" << endl;
		}
	}

	QualType treatType(QualType&& t, SourceRange&& range, bool isInMainFile) {

		const Type* orig = t.getTypePtr();
		const Type* type = nullptr;


		QualType qt = t;

		begin_access:

		type = qt.getTypePtr();

		AccessSpecifier as = AS_public;
<commit_msg>Avoid dereferencing potentially null pointer<commit_after>/*
** SelfPortrait API
** See Copyright Notice in reflection.h
*/
#include "llvm/Support/Host.h"

#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/Tool.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"

#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/Utils.h"
#include "clang/Basic/TargetOptions.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/ASTConsumer.h"
#include "clang/Frontend/ASTUnit.h"
#include "clang/Parse/Parser.h"
#include "clang/Parse/ParseAST.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclTemplate.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Sema/Sema.h"
#include "clang/Sema/Template.h"


#include <vector>
#include <string>
#include <set>
#include <iostream>
#include <fstream>
#include <sstream>
#include <stdexcept>
#include <unordered_map>

#include "definitions.h"
using namespace definitions;

using namespace std;
using namespace clang;
using namespace llvm;

template<class... Args, template<class...> class Associative>
bool contains(const Associative<Args...>& container, const typename Associative<Args...>::key_type& key)
{
	auto it = container.find(key);
	return (it != container.end());
}

template<class... Args, template<class...> class Container>
std::string join(const Container<Args...>& container, const std::string& separator)
{
	if (container.empty()) {
		return "";
	} else {

		auto it = container.begin();

		std::string ret = *it;

		++it;
		for (; it != container.end(); ++it) {
			ret += separator + *it;
		}
		return ret;
	}
}


class MyASTConsumer
{
	TranslationUnit m_tu;
	TranslationUnitBuilder m_builder;

	SourceManager* m_sourceManager;
	PrintingPolicy m_printPol;
	set<Decl*> m_visited;
	Sema& m_sema;
	bool m_instantiateTemplates;


	struct IncompleteType {
		QualType type;
		SourceRange range;
	};

	struct PrivateType {};

	string printType(string orig) {
		string ret;

		for (auto c: orig) {
			if (c == ',') {
				ret += " COMMA ";
			} else {
				ret += c;
			}
		}
		return ret;
	}

	string printType(QualType t) {
		return printType(t.getAsString(m_printPol));
	}

	void treatIncompleteType (IncompleteType t, Decl* decl) {
		SourceLocation start = t.range.getBegin();
		const FileEntry* entry = m_sourceManager->getFileEntryForID(m_sourceManager->getFileID(start));
		int line = m_sourceManager->getSpellingLineNumber(start);
		int column = m_sourceManager->getSpellingColumnNumber(start);



        cerr << (entry ? entry->getName() : "unknown location") << ":" <<
				line << ":" << column <<
				": warning: Incomplete type " <<
				t.type.getAsString(m_printPol) <<

				",\nmeta data will not be emitted for ";

		if (NamedDecl* nd = dyn_cast<NamedDecl>(decl)) {
			cerr << nd->getNameAsString() << endl;
		} else {
			cerr << "the current declaration" << endl;
		}
	}

	QualType treatType(QualType&& t, SourceRange&& range, bool isInMainFile) {

		const Type* orig = t.getTypePtr();
		const Type* type = nullptr;


		QualType qt = t;

		begin_access:

		type = qt.getTypePtr();

		AccessSpecifier as = AS_public;

{'fraction_non_alphanumeric': 0.14819136522753792, 'fraction_numerical': 0.0, 'mean_word_length': 4.547734627831715, 'pattern_counts': {'":': 6, '<': 85, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1740231', 'n_tokens_mistral': 2541, 'n_tokens_neox': 2373, 'n_words': 525}