<commit_before>import * as ts from 'typescript';
import * as util from './util';
import * as path from 'path';
import { Options } from './options';

export interface TsrDeclaration {
  symbol: ts.Symbol,
  name: string
}

// TODO: only scan required nodes and build symbol table for other purposes
export class Scanner {
  private _options: Options;
  private _program: ts.Program;
  private _checker: ts.TypeChecker;

  private declarations: TsrDeclaration[] = [];
  private aliases: Map<ts.Node, ts.Node> = new Map();
  private properties: Map<ts.Node, TypeInfo> = new Map();
  private scanned: Set<ts.Node> = new Set();
  private identifiers: Map<ts.SourceFile, Set<string>> = new Map();
  private symbols: Map<ts.Node, ts.Symbol> = new Map();

  // private skip: ts.SyntaxKind[] = [
  //   ts.SyntaxKind.NamedImports,
  //   ts.SyntaxKind.NamespaceImport,
  //   ts.SyntaxKind.ImportClause,
  //   ts.SyntaxKind.SourceFile,
  //   ts.SyntaxKind.EndOfFileToken,
  //   ts.SyntaxKind.ObjectBindingPattern,
  //   ts.SyntaxKind.ArrayBindingPattern,
  //   ts.SyntaxKind.VariableDeclaration,
  // ];

  private scanNodes: ts.SyntaxKind[] = [
    ts.SyntaxKind.Identifier,
    ts.SyntaxKind.QualifiedName,
    ts.SyntaxKind.ExpressionWithTypeArguments,
    ts.SyntaxKind.ComputedPropertyName,
    ts.SyntaxKind.TypeReference,
    ts.SyntaxKind.MethodSignature,
    ts.SyntaxKind.MethodDeclaration,
    ts.SyntaxKind.CallSignature,
    ts.SyntaxKind.CallExpression,
    ts.SyntaxKind.FunctionDeclaration,
    ts.SyntaxKind.FunctionExpression,
    ts.SyntaxKind.ArrowFunction,
    ts.SyntaxKind.Constructor,
    ts.SyntaxKind.GetAccessor,
    ts.SyntaxKind.SetAccessor,
    ts.SyntaxKind.Parameter,
    ts.SyntaxKind.PropertySignature,
    ts.SyntaxKind.PropertyDeclaration
  ];

  private AllowedDeclarations = ts.SymbolFlags.Interface | ts.SymbolFlags.Class |
  ts.SymbolFlags.RegularEnum | ts.SymbolFlags.ConstEnum | ts.SymbolFlags.Enum |
  ts.SymbolFlags.EnumMember | ts.SymbolFlags.TypeAlias | ts.SymbolFlags.Function |
    /* ts.SymbolFlags.TypeLiteral | */ ts.SymbolFlags.Variable;

  private DisallowedDeclaratins = ts.SymbolFlags.Module | ts.SymbolFlags.TypeParameter |
  ts.SymbolFlags.TypeLiteral;

  constructor(program: ts.Program, options: Options) {
    this._options = options;
    this._program = program;
    this._checker = program.getTypeChecker();
    this.scan();
  }

  public scan(): void {
    const sourceFiles = this.program
      .getSourceFiles()
      .filter(sf => !sf.isDeclarationFile);

    for (let sourceFile of sourceFiles) {<commit_msg>Added missing node kinds to scan.<commit_after>import * as ts from 'typescript';
import * as util from './util';
import * as path from 'path';
import { Options } from './options';

export interface TsrDeclaration {
  symbol: ts.Symbol,
  name: string
}

// TODO: only scan required nodes and build symbol table for other purposes
export class Scanner {
  private _options: Options;
  private _program: ts.Program;
  private _checker: ts.TypeChecker;

  private declarations: TsrDeclaration[] = [];
  private aliases: Map<ts.Node, ts.Node> = new Map();
  private properties: Map<ts.Node, TypeInfo> = new Map();
  private scanned: Set<ts.Node> = new Set();
  private identifiers: Map<ts.SourceFile, Set<string>> = new Map();
  private symbols: Map<ts.Node, ts.Symbol> = new Map();

  // private skip: ts.SyntaxKind[] = [
  //   ts.SyntaxKind.NamedImports,
  //   ts.SyntaxKind.NamespaceImport,
  //   ts.SyntaxKind.ImportClause,
  //   ts.SyntaxKind.SourceFile,
  //   ts.SyntaxKind.EndOfFileToken,
  //   ts.SyntaxKind.ObjectBindingPattern,
  //   ts.SyntaxKind.ArrayBindingPattern,
  //   ts.SyntaxKind.VariableDeclaration,
  // ];

  private scanNodes: ts.SyntaxKind[] = [
    ts.SyntaxKind.Identifier,
    ts.SyntaxKind.QualifiedName,
    ts.SyntaxKind.ExpressionWithTypeArguments,
    ts.SyntaxKind.ComputedPropertyName,
    ts.SyntaxKind.TypeReference,
    ts.SyntaxKind.MethodSignature,
    ts.SyntaxKind.MethodDeclaration,
    ts.SyntaxKind.CallSignature,
    ts.SyntaxKind.CallExpression,
    ts.SyntaxKind.FunctionDeclaration,
    ts.SyntaxKind.FunctionExpression,
    ts.SyntaxKind.ArrowFunction,
    ts.SyntaxKind.Constructor,
    ts.SyntaxKind.GetAccessor,
    ts.SyntaxKind.SetAccessor,
    ts.SyntaxKind.Parameter,
    ts.SyntaxKind.PropertySignature,
    ts.SyntaxKind.PropertyDeclaration,
    ts.SyntaxKind.ConstructSignature,
  ];

  private AllowedDeclarations = ts.SymbolFlags.Interface | ts.SymbolFlags.Class |
  ts.SymbolFlags.RegularEnum | ts.SymbolFlags.ConstEnum | ts.SymbolFlags.Enum |
  ts.SymbolFlags.EnumMember | ts.SymbolFlags.TypeAlias | ts.SymbolFlags.Function |
    /* ts.SymbolFlags.TypeLiteral | */ ts.SymbolFlags.Variable;

  private DisallowedDeclaratins = ts.SymbolFlags.Module | ts.SymbolFlags.TypeParameter |
  ts.SymbolFlags.TypeLiteral;

  constructor(program: ts.Program, options: Options) {
    this._options = options;
    this._program = program;
    this._checker = program.getTypeChecker();
    this.scan();
  }

  public scan(): void {
    const sourceFiles = this.program
      .getSourceFiles()
      .filter(sf => !sf.isDeclarationFile);

    for (let sourceFile of sourceFiles) {
{'fraction_non_alphanumeric': 0.11593924245337435, 'fraction_numerical': 0.0, 'mean_word_length': 3.7637362637362637, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '56200', 'n_tokens_mistral': 1571, 'n_tokens_neox': 1554, 'n_words': 318}