<commit_before>// @ts-check @flow
import { takeRight, dropRight } from 'lodash';
import * as t from 'babel-types';
import template from 'babel-template';
import generate from 'babel-generator';

type JSASTNode = {
  type: string;
  value: string | number | boolean | void;
};

export class ASTStack {
  // We use stack to temporarly store subtree of AST, then you can use popFor() to load subtree in the stack into a container tree node
  jsASTStack: Array<JSASTNode> = [];

  push(subAST: JSASTNode): void {
    this.jsASTStack.push(subAST);
  }

  pop(quentity: number): JSASTNode | Array<JSASTNode> {
    if (this.jsASTStack.length < quentity) {
      throw new Error(`Poping too many sub-AST from JSASTStack, poping ${quentity} but stack with length ${this.jsASTStack.length} is:`, this.jsASTStack);
    }
    if (quentity > 1) {
      const nodes = takeRight(this.jsASTStack, quentity);
      this.jsASTStack = dropRight(this.jsASTStack, quentity);
      return nodes;
    }
    return this.jsASTStack.pop();
  }

  // For debug usage, return generated codes for ASTs
  get codes(): Array<string> {
    return this.jsASTStack
      .map(ast => generate(ast, { quotes: 'single' }))
      .map(output => output.code);
  }
}

export class ASTBuilder extends ASTStack {
  // templateString should contains an 'EXPRESSION'
  unaryExpression(templateString: string): JSASTNode {
    const expression = this.pop(1);
    const buildUnitaryExpression = template(templateString);
    const newUnitaryExpressionStatement = buildUnitaryExpression({ EXPRESSION: expression });
    const newUnitaryExpression = newUnitaryExpressionStatement.expression;
    return newUnitaryExpression;
  }


  // templateString should contains a 'LEFT', a 'RIGHT'
  binaryExpression(templateString: string): JSASTNode {
    const [ left, right ] = this.pop(2);
    const buildBinaryExpression = template(templateString);
    const newBinaryExpressionStatement = buildBinaryExpression({ LEFT: left, RIGHT: right });
    const newBinaryExpression = newBinaryExpressionStatement.expression;
    return newBinaryExpression;
  }

  static iief(ast: any, functionName: string): JSASTNode {
    // 1. if it is a code block, which in JS is a IIEF, or it's a function call
    if (t.isExpression(ast)) {
      const buildIIEF = template(`
        (function ${functionName}() {return null})()
      `);
      const IIEF = buildIIEF({
      }).expression;
      // set return value
      IIEF.callee.body.body[0].argument = ast;
      return IIEF;
    } else
    // 2. if it is a single assignment, how can a humen do this?
    if (t.isVariableDeclaration(ast)) {
      const buildIIEF = template(`
        (function ${functionName}() {STATEMENT})()
      `);
      const IIEF = buildIIEF({
        STATEMENT: ast
      }).expression;
      return IIEF;
    } else {
      // 3. it something I haven't consider
      throw new Error(`${functionName} IIFE building error, ast is `, ast);
    }
  }
}

export default class JSASTBuilder extends ASTBuilder {
  False(): void {
    // booleanLiteral, stringLiteral, and identifier are all extends Expression
    this.push(t.booleanLiteral(false));
  }

  True(): void {
    this.push(t.booleanLiteral(true));
  }

  String(content: string): void {
    // use JSON.parse() to remove "" double quote
    this.push(t.stringLiteral(JSON.parse(content)));
  }

  Int(content: string): void {
    // use JSON.parse() to convert '1' to 1
    this.push(t.numericLiteral(JSON.parse(content)));
  }

  Id(content: string): void {
    this.push(t.identifier(content));
  }

  Parentheses(): void {
    const expression = this.pop(1);
    this.push(t.parenthesizedExpression(expression));
  }

  BoolNot(): void {
    this.push(this.unaryExpression(`
      !EXPRESSION
    `));
  }

  Equal(): void {
    this.push(this.binaryExpression(`
      LEFT === RIGHT
    `));
  }

  LessEqual(): void {
    this.push(this.binaryExpression(`
      LEFT <= RIGHT
    `));
  }

  LessThan(): void {
    this.push(this.binaryExpression(`
      LEFT < RIGHT
    `));
  }

  Negative(): void {
    this.push(this.unaryExpression(`
      -EXPRESSION
    `));
  }

  Division(): void {
    this.push(this.binaryExpression(`
      Math.round( LEFT / RIGHT )
    `));
  }

  Multiply(): void {
    this.push(this.binaryExpression(`
      LEFT * RIGHT
    `));
  }

  Minus(): void {
    this.push(this.binaryExpression(`
      LEFT - RIGHT
    `));
  }

  Add(): void {
    this.push(this.binaryExpression(`
      LEFT + RIGHT
    `));
  }

  Isvoid(): void {
    this.push(this.unaryExpression(`
      !!EXPRESSION
    `));
  }

  NewClass(className: string): void {
    // Not using unaryExpression() since TypeID will not present in the JSAST
    const buildNewClass = template(`
      new CLASS_NAME()
    `);
    const newClassExpressionStatement = buildNewClass({ CLASS_NAME: t.identifier(className) });
    const newClassExpression = newClassExpressionStatement.expression;
    this.push(newClassExpression);
  }

  Case() {
  }

  Block(bodyLength: number): void {
    // 1. If there were only one or less expression, with or without this is the same
    if (bodyLength <= 1) return;
    // 2. If there were more than one expression, execute some first, then return last one
    const expressions = this.pop(bodyLength);
    // 2.1 get lastExpression out
    const lastExpression = expressions.pop();
    // 2.2 get IIEF with 'return lastExpression'
    const IIEF = ASTBuilder.iief(lastExpression, 'codeBlock');
    IIEF.callee.body.body = [...expressions, ...IIEF.callee.body.body];
    this.push(IIEF);
  }

  While(): void {
    const [ test, body ] = this.pop(2);

    this.push(t.whileStatement(test, ASTBuilder.iief(body, 'whileBody')));
  }

  If(): void {
    const [ test, consequent, alternate ] = this.pop(3);

    this.push(t.conditionalExpression(test, ASTBuilder.iief(consequent, 'Ifconsequent'), ASTBuilder.iief(alternate, 'Ifalternate')));
  }

  OwnMethodCall(functionName: string, argumentLength: number): void {
    // Invoke build-in functions inherits from Object type, or own methods
    const functionArguments = [...this.pop(argumentLength)];
    const buildOwnMethodCall = template(`
      this.FUNCTION()
    `);
    const ownMethodCall = buildOwnMethodCall({
      FUNCTION: t.identifier(functionName)
    }).expression;
    ownMethodCall.arguments = functionArguments;
    this.push(ownMethodCall);
  }

  MethodCall(functionName: string, argumentLength: number): void {
    // Invoke someObject.method()
    const functionArguments = [...this.pop(argumentLength)];
    const calleeObject = this.pop(1);
    const buildMethodCall = template(`
      CALLEE.FUNCTION()
    `);
    const methodCall = buildMethodCall({
      CALLEE: calleeObject,
      FUNCTION: t.identifier(functionName)
    }).expression;
    methodCall.arguments = functionArguments;
    this.push(methodCall);
  }

  Assignment(variableName: string): void {
    const expression = this.pop(1);
    const buildAssignment = template(`
      let VARIABLE = EXPRESSION
    `);
    this.push(
      buildAssignment({
        VARIABLE: t.identifier(variableName), EXPRESSION: expression
      })
    );
  }
}
<commit_msg>fix: pop function pop 1 when quentiry === 0<commit_after>// @ts-check @flow
import { takeRight, dropRight } from 'lodash';
import * as t from 'babel-types';
import template from 'babel-template';
import generate from 'babel-generator';

type JSASTNode = {
  type: string;
  value: string | number | boolean | void;
};

export class ASTStack {
  // We use stack to temporarly store subtree of AST, then you can use popFor() to load subtree in the stack into a container tree node
  jsASTStack: Array<JSASTNode> = [];

  push(subAST: JSASTNode): void {
    this.jsASTStack.push(subAST);
  }

  pop(quentity: number): JSASTNode | Array<JSASTNode> {
    if (this.jsASTStack.length < quentity) {
      throw new Error(`Poping too many sub-AST from JSASTStack, poping ${quentity} but stack with length ${this.jsASTStack.length} is:`, this.jsASTStack);
    }
    if (quentity !== 1) {
      const nodes = takeRight(this.jsASTStack, quentity);
      this.jsASTStack = dropRight(this.jsASTStack, quentity);
      return nodes;
    }
    return this.jsASTStack.pop();
  }

  // For debug usage, return generated codes for ASTs
  get codes(): Array<string> {
    return this.jsASTStack
      .map(ast => generate(ast, { quotes: 'single' }))
      .map(output => output.code);
  }
}

export class ASTBuilder extends ASTStack {
  // templateString should contains an 'EXPRESSION'
  unaryExpression(templateString: string): JSASTNode {
    const expression = this.pop(1);
    const buildUnitaryExpression = template(templateString);
    const newUnitaryExpressionStatement = buildUnitaryExpression({ EXPRESSION: expression });
    const newUnitaryExpression = newUnitaryExpressionStatement.expression;
    return newUnitaryExpression;
  }


  // templateString should contains a 'LEFT', a 'RIGHT'
  binaryExpression(templateString: string): JSASTNode {
    const [ left, right ] = this.pop(2);
    const buildBinaryExpression = template(templateString);
    const newBinaryExpressionStatement = buildBinaryExpression({ LEFT: left, RIGHT: right });
    const newBinaryExpression = newBinaryExpressionStatement.expression;
    return newBinaryExpression;
  }

  static iief(ast: any, functionName: string): JSASTNode {
    // 1. if it is a code block, which in JS is a IIEF, or it's a function call
    if (t.isExpression(ast)) {
      const buildIIEF = template(`
        (function ${functionName}() {return null})()
      `);
      const IIEF = buildIIEF({
      }).expression;
      // set return value
      IIEF.callee.body.body[0].argument = ast;
      return IIEF;
    } else
    // 2. if it is a single assignment, how can a humen do this?
    if (t.isVariableDeclaration(ast)) {
      const buildIIEF = template(`
        (function ${functionName}() {STATEMENT})()
      `);
      const IIEF = buildIIEF({
        STATEMENT: ast
      }).expression;
      return IIEF;
    } else {
      // 3. it something I haven't consider
      throw new Error(`${functionName} IIFE building error, ast is `, ast);
    }
  }
}

export default class JSASTBuilder extends ASTBuilder {
  False(): void {
    // booleanLiteral, stringLiteral, and identifier are all extends Expression
    this.push(t.booleanLiteral(false));
  }

  True(): void {
    this.push(t.booleanLiteral(true));
  }

  String(content: string): void {
    // use JSON.parse() to remove "" double quote
    this.push(t.stringLiteral(JSON.parse(content)));
  }

  Int(content: string): void {
    // use JSON.parse() to convert '1' to 1
    this.push(t.numericLiteral(JSON.parse(content)));
  }

  Id(content: string): void {
    this.push(t.identifier(content));
  }

  Parentheses(): void {
    const expression = this.pop(1);
    this.push(t.parenthesizedExpression(expression));
  }

  BoolNot(): void {
    this.push(this.unaryExpression(`
      !EXPRESSION
    `));
  }

  Equal(): void {
    this.push(this.binaryExpression(`
      LEFT === RIGHT
    `));
  }

  LessEqual(): void {
    this.push(this.binaryExpression(`
      LEFT <= RIGHT
    `));
  }

  LessThan(): void {
    this.push(this.binaryExpression(`
      LEFT < RIGHT
    `));
  }

  Negative(): void {
    this.push(this.unaryExpression(`
      -EXPRESSION
    `));
  }

  Division(): void {
    this.push(this.binaryExpression(`
      Math.round( LEFT / RIGHT )
    `));
  }

  Multiply(): void {
    this.push(this.binaryExpression(`
      LEFT * RIGHT
    `));
  }

  Minus(): void {
    this.push(this.binaryExpression(`
      LEFT - RIGHT
    `));
  }

  Add(): void {
    this.push(this.binaryExpression(`
      LEFT + RIGHT
    `));
  }

  Isvoid(): void {
    this.push(this.unaryExpression(`
      !!EXPRESSION
    `));
  }

  NewClass(className: string): void {
    // Not using unaryExpression() since TypeID will not present in the JSAST
    const buildNewClass = template(`
      new CLASS_NAME()
    `);
    const newClassExpressionStatement = buildNewClass({ CLASS_NAME: t.identifier(className) });
    const newClassExpression = newClassExpressionStatement.expression;
    this.push(newClassExpression);
  }

  Case() {
  }

  Block(bodyLength: number): void {
    // 1. If there were only one or less expression, with or without this is the same
    if (bodyLength <= 1) return;
    // 2. If there were more than one expression, execute some first, then return last one
    const expressions = this.pop(bodyLength);
    // 2.1 get lastExpression out
    const lastExpression = expressions.pop();
    // 2.2 get IIEF with 'return lastExpression'
    const IIEF = ASTBuilder.iief(lastExpression, 'codeBlock');
    IIEF.callee.body.body = [...expressions, ...IIEF.callee.body.body];
    this.push(IIEF);
  }

  While(): void {
    const [ test, body ] = this.pop(2);

    this.push(t.whileStatement(test, ASTBuilder.iief(body, 'whileBody')));
  }

  If(): void {
    const [ test, consequent, alternate ] = this.pop(3);

    this.push(t.conditionalExpression(test, ASTBuilder.iief(consequent, 'Ifconsequent'), ASTBuilder.iief(alternate, 'Ifalternate')));
  }

  OwnMethodCall(functionName: string, argumentLength: number): void {
    // Invoke build-in functions inherits from Object type, or own methods
    const functionArguments = [...this.pop(argumentLength)];
    const buildOwnMethodCall = template(`
      this.FUNCTION()
    `);
    const ownMethodCall = buildOwnMethodCall({
      FUNCTION: t.identifier(functionName)
    }).expression;
    ownMethodCall.arguments = functionArguments;
    this.push(ownMethodCall);
  }

  MethodCall(functionName: string, argumentLength: number): void {
    // Invoke someObject.method()
    const functionArguments = [...this.pop(argumentLength)];
    const calleeObject = this.pop(1);
    const buildMethodCall = template(`
      CALLEE.FUNCTION()
    `);
    const methodCall = buildMethodCall({
      CALLEE: calleeObject,
      FUNCTION: t.identifier(functionName)
    }).expression;
    methodCall.arguments = functionArguments;
    this.push(methodCall);
  }

  Assignment(variableName: string): void {
    const expression = this.pop(1);
    const buildAssignment = template(`
      let VARIABLE = EXPRESSION
    `);
    this.push(
      buildAssignment({
        VARIABLE: t.identifier(variableName), EXPRESSION: expression
      })
    );
  }
}

{'fraction_non_alphanumeric': 0.12525980324234445, 'fraction_numerical': 0.003048358043508383, 'mean_word_length': 3.1515674431981595, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2715261', 'n_tokens_mistral': 4473, 'n_tokens_neox': 4291, 'n_words': 1121}