<commit_before>import * as fs from 'fs';
import * as ts from 'typescript';

interface ClassData {
  name: string;
  class: ts.VariableDeclaration;
  return: ts.ReturnStatement;
}

export interface OpDesc {
  op: 'insert' | 'remove';
  pos: number;
  text: string;
}

export function foldFile(file: string, name: string): string {
  let contents = fs.readFileSync(file).toString();

  const options: ts.CompilerOptions = {
    allowJs: true,
  };

  const program = ts.createProgram([file], options);
  const source = program.getSourceFile(file);
  const checker = program.getTypeChecker();

  const ops = foldAllClasses(source, checker);
  console.error(`${name}: folding ${ops.length} ops`);

  ops.sort((a, b) => {
    if (a.pos < b.pos) {
      return 1;
    } else if (a.pos === b.pos) {
      return 0;
    } else {
      return -1;
    }
  });

  ops.forEach(op => {
    const prefix = contents.substring(0, op.pos);
    switch (op.op) {
      case 'insert':
        const suffix = contents.substring(op.pos);
        contents = prefix + op.text + suffix;
        break;
      case 'remove':
        const remainder = contents.substring(op.pos + op.text.length);
        contents = prefix + remainder;
        break;
    }
  });

  return contents;
}

export function foldAllClasses(node: ts.Node, checker: ts.TypeChecker): OpDesc[] {
  const classes: ClassData[] = [];
  ts.forEachChild(node, child => {
    if (child.kind !== ts.SyntaxKind.VariableStatement) {
      return;
    }
    const varStmt = child as ts.VariableStatement;
    if (varStmt.declarationList.declarations.length > 1) {
      return;
    }
    const varDecl = varStmt.declarationList.declarations[0];
    if (varDecl.name.kind !== ts.SyntaxKind.Identifier) {
      return;
    }
    const name = (varDecl.name as ts.Identifier).text;
    const expr = varDecl.initializer;
    if (!expr || expr.kind !== ts.SyntaxKind.ParenthesizedExpression) {
      return;
    }
    if ((expr as ts.ParenthesizedExpression).expression.kind !== ts.SyntaxKind.CallExpression) {
      return;
    }
    const callExpr = (expr as ts.ParenthesizedExpression).expression as ts.CallExpression;
    if (callExpr.expression.kind !== ts.SyntaxKind.FunctionExpression) {
      return;
    }
    const fn = callExpr.expression as ts.FunctionExpression;
    if (fn.body.statements.length < 2) {
      return;
    }
    if (fn.body.statements[0].kind !== ts.SyntaxKind.FunctionDeclaration) {
      return;
    }<commit_msg>add some comments to class-fold<commit_after>import * as fs from 'fs';
import * as ts from 'typescript';

interface ClassData {
  name: string;
  class: ts.VariableDeclaration;
  return: ts.ReturnStatement;
}

export interface OpDesc {
  op: 'insert' | 'remove';
  pos: number;
  text: string;
}

export function foldFile(file: string, name: string): string {
  let contents = fs.readFileSync(file).toString();

  const options: ts.CompilerOptions = {
    allowJs: true,
  };

  const program = ts.createProgram([file], options);
  const source = program.getSourceFile(file);
  const checker = program.getTypeChecker();

  // Get all class fold operations for this file.
  const ops = foldAllClasses(source, checker);
  console.error(`${name}: folding ${ops.length} ops`);

  // Sort operations by position.
  ops.sort((a, b) => {
    if (a.pos < b.pos) {
      return 1;
    } else if (a.pos === b.pos) {
      return 0;
    } else {
      return -1;
    }
  });

  ops.forEach(op => {
    const prefix = contents.substring(0, op.pos);
    // This works because the amount of text inserted is always equal to the removed.
    // Otherwise, the positions would affect each other.
    switch (op.op) {
      case 'insert':
        const suffix = contents.substring(op.pos);
        contents = prefix + op.text + suffix;
        break;
      case 'remove':
        const remainder = contents.substring(op.pos + op.text.length);
        contents = prefix + remainder;
        break;
    }
  });

  return contents;
}

export function foldAllClasses(node: ts.Node, checker: ts.TypeChecker): OpDesc[] {
  const classes: ClassData[] = [];
  // Find all class declarations, build a ClassData for each.
  ts.forEachChild(node, child => {
    if (child.kind !== ts.SyntaxKind.VariableStatement) {
      return;
    }
    const varStmt = child as ts.VariableStatement;
    if (varStmt.declarationList.declarations.length > 1) {
      return;
    }
    const varDecl = varStmt.declarationList.declarations[0];
    if (varDecl.name.kind !== ts.SyntaxKind.Identifier) {
      return;
    }
    const name = (varDecl.name as ts.Identifier).text;
    const expr = varDecl.initializer;
    if (!expr || expr.kind !== ts.SyntaxKind.ParenthesizedExpression) {
      return;
    }
    if ((expr as ts.ParenthesizedExpression).expression.kind !== ts.SyntaxKind.CallExpression) {
      return;
    }
    const callExpr = (expr as ts.ParenthesizedExpression).expression as ts.CallExpression;
    if (callExpr.expression.kind !== ts.SyntaxKind.FunctionExpression) {
      return;
    }
    const fn = callExpr.expression as ts.FunctionExpression;
    if (fn.body.statements.length < 2) {
      return;
    }
    if (fn.body.statements[0].kind !== ts.SyntaxKind.FunctionDeclaration) {
      return;
    }
{'fraction_non_alphanumeric': 0.12944983818770225, 'fraction_numerical': 0.0030458785455929945, 'mean_word_length': 2.9267563527653215, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6841420', 'n_tokens_mistral': 1619, 'n_tokens_neox': 1604, 'n_words': 408}