<commit_before>import * as ts from 'typescript';

export function formatDiagnostics(diags: ts.Diagnostic[]): string {
  return diags.map((d) => {
                let res = ts.DiagnosticCategory[d.category];
                if (d.file) res += ' at ' + d.file.fileName + ':';
                if (d.start) {
                  let {line, character} = d.file.getLineAndCharacterOfPosition(d.start);
                  res += line + ':' + character + ':';
                }
                res += ' ' + d.messageText;
                return res;
              })
      .join('\n');
}

export type StringMap = {
  [fileName: string]: string
};

export type AnnotatedProgram = StringMap;

const VISIBILITY_FLAGS = ts.NodeFlags.Private | ts.NodeFlags.Protected | ts.NodeFlags.Public;

/**
 * A source processor that takes TypeScript code and annotates the output with Closure-style JSDoc
 * comments.
 */
class Annotator {
  private output: string[];

  constructor() {}

  annotate(args: string[]): AnnotatedProgram {
    let tsArgs = ts.parseCommandLine(args);
    if (tsArgs.errors) {
      this.fail(formatDiagnostics(tsArgs.errors));
    }
    let program = ts.createProgram(tsArgs.fileNames, tsArgs.options);
    let diags = ts.getPreEmitDiagnostics(program);
    if (diags && diags.length) this.fail(formatDiagnostics(diags));
    return this.annotateProgram(program);
  }

  annotateProgram(program: ts.Program): AnnotatedProgram {
    let res: AnnotatedProgram = {};
    for (let sf of program.getSourceFiles()) {
      if (sf.fileName.match(/\.d\.ts$/)) continue;
      this.output = [];
      this.visit(sf);
      res[sf.fileName] = this.output.join('');
    }
    return res;
  }

  private emit(str: string) { this.output.push(str); }

  private visit(node: ts.Node) {
    // console.log('node:', (<any>ts).SyntaxKind[node.kind]);
    switch (node.kind) {
      case ts.SyntaxKind.VariableDeclaration:
        this.maybeVisitType((<ts.VariableDeclaration>node).type);
        this.writeNode(node);
        break;
      case ts.SyntaxKind.ClassDeclaration: {
        let classNode = <ts.ClassDeclaration>node;
        let hasCtor = classNode.members.some((e) => e.kind === ts.SyntaxKind.Constructor);
        if (hasCtor) {
          this.writeNode(classNode);
          break;
        }
        // Emit a synthetic ctor.
        // TODO(martinprobst): Handle inherited parent ctors.
        this.writeTextBetween(classNode, classNode.getLastToken());
        this.emit('constructor() {\n');
        this.emitStubDeclarations(classNode, []);
        this.emit('}\n');
        this.writeNode(classNode.getLastToken());
        break;
      }
      case ts.SyntaxKind.Constructor: {
        let ctor = <ts.ConstructorDeclaration>node;
        this.writeTextBetween(ctor, ctor.body);
        if (ctor.body.statements.length) {
          // Insert before the first code in the ctor.
          this.writeTextBetween(ctor.body, ctor.body.statements[0]);
        } else {
          // Empty ctor - just insert before the end of it.
          this.writeTextBetween(ctor.body, ctor.body.getLastToken());
        }

        let paramProps = ctor.parameters.filter((p) => !!(p.flags & VISIBILITY_FLAGS));
        this.emitStubDeclarations(<ts.ClassLikeDeclaration>ctor.parent, paramProps);

        if (ctor.body.statements.length) {
          let firstStmt = ctor.body.statements[0];
          this.emit(
              ctor.body.getSourceFile().getText().substring(
                  firstStmt.getFullStart(), ctor.body.getEnd()));
        } else {
          let remaining = ctor.getSourceFile().getText().substring(
              ctor.body.getLastToken().getFullStart(), ctor.body.getEnd());
          this.emit(remaining);
        }

        break;
      }
      case ts.SyntaxKind.FunctionDeclaration:
      case ts.SyntaxKind.ArrowFunction:
        let fnDecl = <ts.FunctionLikeDeclaration>node;
        this.maybeVisitType(fnDecl.type, '@return');
        let writeOffset = fnDecl.getFullStart();
        // Parameters.
        if (fnDecl.parameters.length) {
          for (let param of fnDecl.parameters) {
            this.writeTextFromOffset(writeOffset, param);
            writeOffset = param.getEnd();
            this.maybeVisitType(param.type);
            this.visit(param);
          }
        }
        // Return type.
        if (fnDecl.type) {
          this.writeTextFromOffset(writeOffset, fnDecl.type);
          this.visit(fnDecl.type);
          writeOffset = fnDecl.type.getEnd();
        }
        // Body.
        this.writeTextFromOffset(writeOffset, fnDecl.body);
        this.visit(fnDecl.body);
        break;
      default:
        this.writeNode(node);
        break;
    }
  }

  private emitStubDeclarations(
      classDecl: ts.ClassLikeDeclaration, paramProps: ts.ParameterDeclaration[]) {<commit_msg>improve logging of visited nodes<commit_after>import * as ts from 'typescript';

export function formatDiagnostics(diags: ts.Diagnostic[]): string {
  return diags.map((d) => {
                let res = ts.DiagnosticCategory[d.category];
                if (d.file) res += ' at ' + d.file.fileName + ':';
                if (d.start) {
                  let {line, character} = d.file.getLineAndCharacterOfPosition(d.start);
                  res += line + ':' + character + ':';
                }
                res += ' ' + d.messageText;
                return res;
              })
      .join('\n');
}

export type StringMap = {
  [fileName: string]: string
};

export type AnnotatedProgram = StringMap;

const VISIBILITY_FLAGS = ts.NodeFlags.Private | ts.NodeFlags.Protected | ts.NodeFlags.Public;

/**
 * A source processor that takes TypeScript code and annotates the output with Closure-style JSDoc
 * comments.
 */
class Annotator {
  private output: string[];
  private indent: number;

  constructor() { this.indent = 0; }

  annotate(args: string[]): AnnotatedProgram {
    let tsArgs = ts.parseCommandLine(args);
    if (tsArgs.errors) {
      this.fail(formatDiagnostics(tsArgs.errors));
    }
    let program = ts.createProgram(tsArgs.fileNames, tsArgs.options);
    let diags = ts.getPreEmitDiagnostics(program);
    if (diags && diags.length) this.fail(formatDiagnostics(diags));
    return this.annotateProgram(program);
  }

  annotateProgram(program: ts.Program): AnnotatedProgram {
    let res: AnnotatedProgram = {};
    for (let sf of program.getSourceFiles()) {
      if (sf.fileName.match(/\.d\.ts$/)) continue;
      this.output = [];
      this.visit(sf);
      assert(this.indent == 0, 'visit() failed to track nesting');
      res[sf.fileName] = this.output.join('');
    }
    return res;
  }

  private emit(str: string) { this.output.push(str); }

  private logWithIndent(message: string) {
    let prefix = new Array(this.indent + 1).join('| ');
    console.log(prefix + message);
  }

  private visit(node: ts.Node) {
    this.indent++;
    // this.logWithIndent('node: ' + (<any>ts).SyntaxKind[node.kind]);
    switch (node.kind) {
      case ts.SyntaxKind.VariableDeclaration:
        this.maybeVisitType((<ts.VariableDeclaration>node).type);
        this.writeNode(node);
        break;
      case ts.SyntaxKind.ClassDeclaration: {
        let classNode = <ts.ClassDeclaration>node;
        let hasCtor = classNode.members.some((e) => e.kind === ts.SyntaxKind.Constructor);
        if (hasCtor) {
          this.writeNode(classNode);
          break;
        }
        // Emit a synthetic ctor.
        // TODO(martinprobst): Handle inherited parent ctors.
        this.writeTextBetween(classNode, classNode.getLastToken());
        this.emit('constructor() {\n');
        this.emitStubDeclarations(classNode, []);
        this.emit('}\n');
        this.writeNode(classNode.getLastToken());
        break;
      }
      case ts.SyntaxKind.Constructor: {
        let ctor = <ts.ConstructorDeclaration>node;
        this.writeTextBetween(ctor, ctor.body);
        if (ctor.body.statements.length) {
          // Insert before the first code in the ctor.
          this.writeTextBetween(ctor.body, ctor.body.statements[0]);
        } else {
          // Empty ctor - just insert before the end of it.
          this.writeTextBetween(ctor.body, ctor.body.getLastToken());
        }

        let paramProps = ctor.parameters.filter((p) => !!(p.flags & VISIBILITY_FLAGS));
        this.emitStubDeclarations(<ts.ClassLikeDeclaration>ctor.parent, paramProps);

        if (ctor.body.statements.length) {
          let firstStmt = ctor.body.statements[0];
          this.emit(
              ctor.body.getSourceFile().getText().substring(
                  firstStmt.getFullStart(), ctor.body.getEnd()));
        } else {
          let remaining = ctor.getSourceFile().getText().substring(
              ctor.body.getLastToken().getFullStart(), ctor.body.getEnd());
          this.emit(remaining);
        }

        break;
      }
      case ts.SyntaxKind.FunctionDeclaration:
      case ts.SyntaxKind.ArrowFunction:
        let fnDecl = <ts.FunctionLikeDeclaration>node;
        this.maybeVisitType(fnDecl.type, '@return');
        let writeOffset = fnDecl.getFullStart();
        // Parameters.
        if (fnDecl.parameters.length) {
          for (let param of fnDecl.parameters) {
            this.writeTextFromOffset(writeOffset, param);
            writeOffset = param.getEnd();
            this.maybeVisitType(param.type);
            this.visit(param);
          }
        }
        // Return type.
        if (fnDecl.type) {
          this.writeTextFromOffset(writeOffset, fnDecl.type);
          this.visit(fnDecl.type);
          writeOffset = fnDecl.type.getEnd();
        }
        // Body.
        this.writeTextFromOffset(writeOffset, fnDecl.body);
        this.visit(fnDecl.body);
        break;
      default:
        this.writeNode(node);
        break;
    }
    this.indent--;
  }

  private emitStubDeclarations(
      classDecl: ts.ClassLikeDeclaration, paramProps: ts.ParameterDeclaration[]) {
{'fraction_non_alphanumeric': 0.13108839446782922, 'fraction_numerical': 0.0007015433954700341, 'mean_word_length': 2.3441689008042896, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5165304', 'n_tokens_mistral': 2979, 'n_tokens_neox': 2915, 'n_words': 554}