<commit_before>import * as ts from 'typescript';

import { compileDts } from './compile-dts';
import { TypesUsageEvaluator } from './types-usage-evaluator';
import {
	hasNodeModifier,
	isNodeNamedDeclaration,
} from './typescript-helpers';

import {
	getLibraryName,
	getTypesLibraryName,
} from './node-modules-helpers';

import {
	normalLog,
	verboseLog,
} from './logger';

export interface GenerationOptions {
	outputFilenames?: boolean;
	failOnClass?: boolean;
	inlinedLibraries?: string[];
	importedLibraries?: string[];
	allowedTypesLibraries?: string[];
}

const skippedNodes = [
	ts.SyntaxKind.ExportDeclaration,
	ts.SyntaxKind.ExportAssignment,
	ts.SyntaxKind.ImportDeclaration,
	ts.SyntaxKind.ImportEqualsDeclaration,
];

// tslint:disable-next-line:cyclomatic-complexity
export function generateDtsBundle(filePath: string, options: GenerationOptions = {}): string {
	const inlinedLibraries = options.inlinedLibraries || [];
	const importedLibraries = options.importedLibraries || [];
	const allowedTypesLibs = options.allowedTypesLibraries;

	if (!ts.sys.fileExists(filePath)) {
		throw new Error(`File "${filePath}" does not exist`);
	}

	const program = compileDts(filePath);
	const typeChecker = program.getTypeChecker();

	// we do not need all files from node_modules dir
	const sourceFiles = program.getSourceFiles().filter((file: ts.SourceFile) => {
		const fileName = file.fileName;
		const libraryName = getLibraryName(fileName);
		if (libraryName === null) {
			return true;
		}

		const typesLibName = getTypesLibraryName(fileName);
		if (typesLibName !== null && (allowedTypesLibs === undefined || allowedTypesLibs.indexOf(typesLibName) !== -1)) {
			return true;
		}

		return inlinedLibraries.indexOf(libraryName) !== -1 || importedLibraries.indexOf(libraryName) !== -1;
	});

	verboseLog(`Input source files:\n  ${sourceFiles.map((file: ts.SourceFile) => file.fileName).join('\n  ')}`);

	const typesUsageEvaluator = new TypesUsageEvaluator(sourceFiles, typeChecker);

	const rootSourceFileSymbol = typeChecker.getSymbolAtLocation(getRootSourceFile(program));
	if (rootSourceFileSymbol === undefined) {
		throw new Error('Symbol for root source file not found');
	}

	const rootFileExports = typeChecker.getExportsOfModule(rootSourceFileSymbol).map((symbol: ts.Symbol) => {
		if (symbol.flags & ts.SymbolFlags.Alias) {
			// so we need to have original symbols from source file
			symbol = typeChecker.getAliasedSymbol(symbol);
		}

		return symbol;
	});

	const usedTypes = new Set<string>();
	const importedSymbols = new Map<string, Set<string>>();

	let resultOutput = '';
	for (const sourceFile of sourceFiles) {
		verboseLog(`\n\n======= Preparing file: ${sourceFile.fileName} =======`);

		const sourceFileText = sourceFile.getFullText();

		let fileOutput = '';
		for (const node of sourceFile.statements) {
			// we should skip import and exports statements
			if (skippedNodes.indexOf(node.kind) !== -1) {
				continue;
			}

			let isNodeUsed = false;
			if (isNodeNamedDeclaration(node)) {
				isNodeUsed = rootFileExports.some(typesUsageEvaluator.isTypeUsedBySymbol.bind(typesUsageEvaluator, node));
			} else if (node.kind === ts.SyntaxKind.VariableStatement) {
				const declarations = (node as ts.VariableStatement).declarationList.declarations;
				isNodeUsed = declarations.some((declaration: ts.VariableDeclaration) => {
					return isDeclarationExported(rootFileExports, typeChecker, declaration);
				});
			}

			if (!isNodeUsed) {
				verboseLog(`Skip file member: ${node.getText().replace(/(\n|\r)/g, '').slice(0, 50)}...`);
				continue;
			}

			const typesLibraryName = getTypesLibraryName(sourceFile.fileName);
			if (typesLibraryName !== null) {
				if (!usedTypes.has(typesLibraryName)) {
					normalLog(`Library "${typesLibraryName}" will be added via reference directive`);
					usedTypes.add(typesLibraryName);
				}

				break;
			}

			const importedLibraryName = getLibraryName(sourceFile.fileName);
			if (importedLibraryName !== null && importedLibraries.indexOf(importedLibraryName) !== -1) {
				const nodeIdentifier = (node as ts.DeclarationStatement).name;
				if (nodeIdentifier === undefined) {
					throw new Error(`Import/usage unnamed declaration: ${node.getText()}`);<commit_msg>Improved checking of libraries for import/types usage (calc once for source file)<commit_after>import * as ts from 'typescript';

import { compileDts } from './compile-dts';
import { TypesUsageEvaluator } from './types-usage-evaluator';
import {
	hasNodeModifier,
	isNodeNamedDeclaration,
} from './typescript-helpers';

import {
	getLibraryName,
	getTypesLibraryName,
} from './node-modules-helpers';

import {
	normalLog,
	verboseLog,
} from './logger';

export interface GenerationOptions {
	outputFilenames?: boolean;
	failOnClass?: boolean;
	inlinedLibraries?: string[];
	importedLibraries?: string[];
	allowedTypesLibraries?: string[];
}

const skippedNodes = [
	ts.SyntaxKind.ExportDeclaration,
	ts.SyntaxKind.ExportAssignment,
	ts.SyntaxKind.ImportDeclaration,
	ts.SyntaxKind.ImportEqualsDeclaration,
];

// tslint:disable-next-line:cyclomatic-complexity
export function generateDtsBundle(filePath: string, options: GenerationOptions = {}): string {
	const inlinedLibraries = options.inlinedLibraries || [];
	const importedLibraries = options.importedLibraries || [];
	const allowedTypesLibs = options.allowedTypesLibraries;

	if (!ts.sys.fileExists(filePath)) {
		throw new Error(`File "${filePath}" does not exist`);
	}

	const program = compileDts(filePath);
	const typeChecker = program.getTypeChecker();

	// we do not need all files from node_modules dir
	const sourceFiles = program.getSourceFiles().filter((file: ts.SourceFile) => {
		const fileName = file.fileName;
		const libraryName = getLibraryName(fileName);
		if (libraryName === null) {
			return true;
		}

		const typesLibName = getTypesLibraryName(fileName);
		if (typesLibName !== null && (allowedTypesLibs === undefined || allowedTypesLibs.indexOf(typesLibName) !== -1)) {
			return true;
		}

		return inlinedLibraries.indexOf(libraryName) !== -1 || importedLibraries.indexOf(libraryName) !== -1;
	});

	verboseLog(`Input source files:\n  ${sourceFiles.map((file: ts.SourceFile) => file.fileName).join('\n  ')}`);

	const typesUsageEvaluator = new TypesUsageEvaluator(sourceFiles, typeChecker);

	const rootSourceFileSymbol = typeChecker.getSymbolAtLocation(getRootSourceFile(program));
	if (rootSourceFileSymbol === undefined) {
		throw new Error('Symbol for root source file not found');
	}

	const rootFileExports = typeChecker.getExportsOfModule(rootSourceFileSymbol).map((symbol: ts.Symbol) => {
		if (symbol.flags & ts.SymbolFlags.Alias) {
			// so we need to have original symbols from source file
			symbol = typeChecker.getAliasedSymbol(symbol);
		}

		return symbol;
	});

	const usedTypes = new Set<string>();
	const importedSymbols = new Map<string, Set<string>>();

	let resultOutput = '';
	for (const sourceFile of sourceFiles) {
		verboseLog(`\n\n======= Preparing file: ${sourceFile.fileName} =======`);

		const sourceFileText = sourceFile.getFullText();

		const typesLibraryName = getTypesLibraryName(sourceFile.fileName);
		const importedLibraryName = getLibraryName(sourceFile.fileName);
		const isAllowedAsImportedLibrary = importedLibraryName !== null && importedLibraries.indexOf(importedLibraryName) !== -1;

		let fileOutput = '';
		for (const node of sourceFile.statements) {
			// we should skip import and exports statements
			if (skippedNodes.indexOf(node.kind) !== -1) {
				continue;
			}

			let isNodeUsed = false;
			if (isNodeNamedDeclaration(node)) {
				isNodeUsed = rootFileExports.some(typesUsageEvaluator.isTypeUsedBySymbol.bind(typesUsageEvaluator, node));
			} else if (node.kind === ts.SyntaxKind.VariableStatement) {
				const declarations = (node as ts.VariableStatement).declarationList.declarations;
				isNodeUsed = declarations.some((declaration: ts.VariableDeclaration) => {
					return isDeclarationExported(rootFileExports, typeChecker, declaration);
				});
			}

			if (!isNodeUsed) {
				verboseLog(`Skip file member: ${node.getText().replace(/(\n|\r)/g, '').slice(0, 50)}...`);
				continue;
			}

			if (typesLibraryName !== null) {
				if (!usedTypes.has(typesLibraryName)) {
					normalLog(`Library "${typesLibraryName}" will be added via reference directive`);
					usedTypes.add(typesLibraryName);
				}

				break;
			}

			if (importedLibraryName !== null && isAllowedAsImportedLibrary) {
				const nodeIdentifier = (node as ts.DeclarationStatement).name;
				if (nodeIdentifier === undefined) {
					throw new Error(`Import/usage unnamed declaration: ${node.getText()}`);
{'fraction_non_alphanumeric': 0.13079584775086506, 'fraction_numerical': 0.001845444059976932, 'mean_word_length': 4.988259668508287, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1323944', 'n_tokens_mistral': 2765, 'n_tokens_neox': 2554, 'n_words': 577}