<commit_before>import { isArray, camelCase } from './utils';
import {
  block_statement,
  empty_statement,
  infix_statement,
  multi_line_statement,
  token_statement
} from './statement';

function letToVar([cmd, ...vars], options) {
  const varPairs = [];
  for (let i = 0; i < vars.length; i += 2) {
    let name = vars[ i ];
    let val = vars[ i + 1 ];

    if (name && val) {
      varPairs.push(infix_statement({
        statements: [ token_statement({ token: `var ${name}`} ), toJsTree(val, { embedded: true }) ],
        separator: ' = ',
        terminate: true
      }));
    }
  }

  if (varPairs.length < 1) {
    return empty_statement();
  }
  else {
    return multi_line_statement(merge(options, {
      statements: varPairs
    }));
  }
}

function math_operator([operator, ...params], options) {
  return infix_statement(merge(options, {
    statements: params.map(param => toJsTree(param, { embedded: true })),
    separator: ` ${operator} `
  }));
}

function define_function([operator, name, params, ...body], options) {
  if (isArray(name)) {
    body.unshift(params);
    params = name;
    name = "";
  }
  params = params.join(', ');

  let final = body.pop();
  if (final) {
    final = toJsTree(final, { returnStatement: true, terminate: true });
  } else {
    final = token_statement({ token: 'null', returnStatement: true, terminate: true });
  }

  let statements = body.map(toJsTree, { terminate: true });
  statements.push(final);

  return block_statement(merge(options, {
    openBlock: `function ${name}(${params}) {`,
    statements,
    closeBlock: `}`
  }));
}

function define_module([operator, name, ...body], options) {
  const statements = body.map(toJsTree, { terminate: true });

  return block_statement(merge(options, {
    openBlock: `module('${name}', function (require, export) {`,
    statements,
    closeBlock: `})`
  }));
}

function module_require([operator, moduleName, reqTokens], options) {
  const statements = reqTokens.map(token =>
    `var ${token} = ${moduleName}['${token}']`);
  statements.unshift(`var ${moduleName} = require('${moduleName}')`);

  return statements
    .map(s => s + ';')
    .join(' ');
}

function module_export([operator, moduleName, statement], options) {
  return `export('${moduleName}', ${toJsTree(statement, { embedded: true })})`;
}

function function_call([fn, ...params], options) {
  const evaluated_params = params.map(p => toJsTree(p, { embedded: true }));

  return `${fn}( ${evaluated_params.join(', ')} )`;
}

export const toJsTree = function (arr, options = { }) {
  if (!isArray(arr)) {
    return token_statement({ token: arr });
  }

  switch(arr[0]) {
    case 'function': return define_function(arr, options);
    case 'module': return define_module(arr, options);
    case 'require': return module_require(arr, options);
    case 'export': return module_export(arr, options);
    case 'let': return letToVar(arr, options);
    case '+': return math_operator(arr, options);
    case '-': return math_operator(arr, options);
    case '*': return math_operator(arr, options);
    case '/': return math_operator(arr, options);
  }

  return function_call(arr, options);
};
<commit_msg>statement intention wasn't clear, adjusting whitespace to parse easier<commit_after>import { isArray, camelCase } from './utils';
import {
  block_statement,
  empty_statement,
  infix_statement,
  multi_line_statement,
  token_statement
} from './statement';

function letToVar([cmd, ...vars], options) {
  const varPairs = [];
  for (let i = 0; i < vars.length; i += 2) {
    let name = vars[ i ];
    let val = vars[ i + 1 ];

    if (name && val) {
      varPairs.push(infix_statement({
        statements: [
          token_statement({ token: `var ${name}`} ),
          toJsTree(val, { embedded: true })
        ],
        separator: ' = ',
        terminate: true
      }));
    }
  }

  if (varPairs.length < 1) {
    return empty_statement();
  }
  else {
    return multi_line_statement(merge(options, {
      statements: varPairs
    }));
  }
}

function math_operator([operator, ...params], options) {
  return infix_statement(merge(options, {
    statements: params.map(param => toJsTree(param, { embedded: true })),
    separator: ` ${operator} `
  }));
}

function define_function([operator, name, params, ...body], options) {
  if (isArray(name)) {
    body.unshift(params);
    params = name;
    name = "";
  }
  params = params.join(', ');

  let final = body.pop();
  if (final) {
    final = toJsTree(final, { returnStatement: true, terminate: true });
  } else {
    final = token_statement({ token: 'null', returnStatement: true, terminate: true });
  }

  let statements = body.map(toJsTree, { terminate: true });
  statements.push(final);

  return block_statement(merge(options, {
    openBlock: `function ${name}(${params}) {`,
    statements,
    closeBlock: `}`
  }));
}

function define_module([operator, name, ...body], options) {
  const statements = body.map(toJsTree, { terminate: true });

  return block_statement(merge(options, {
    openBlock: `module('${name}', function (require, export) {`,
    statements,
    closeBlock: `})`
  }));
}

function module_require([operator, moduleName, reqTokens], options) {
  const statements = reqTokens.map(token =>
    `var ${token} = ${moduleName}['${token}']`);
  statements.unshift(`var ${moduleName} = require('${moduleName}')`);

  return statements
    .map(s => s + ';')
    .join(' ');
}

function module_export([operator, moduleName, statement], options) {
  return `export('${moduleName}', ${toJsTree(statement, { embedded: true })})`;
}

function function_call([fn, ...params], options) {
  const evaluated_params = params.map(p => toJsTree(p, { embedded: true }));

  return `${fn}( ${evaluated_params.join(', ')} )`;
}

export const toJsTree = function (arr, options = { }) {
  if (!isArray(arr)) {
    return token_statement({ token: arr });
  }

  switch(arr[0]) {
    case 'function': return define_function(arr, options);
    case 'module': return define_module(arr, options);
    case 'require': return module_require(arr, options);
    case 'export': return module_export(arr, options);
    case 'let': return letToVar(arr, options);
    case '+': return math_operator(arr, options);
    case '-': return math_operator(arr, options);
    case '*': return math_operator(arr, options);
    case '/': return math_operator(arr, options);
  }

  return function_call(arr, options);
};

{'fraction_non_alphanumeric': 0.1703509043128768, 'fraction_numerical': 0.001545833977430824, 'mean_word_length': 3.73992673992674, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '500796', 'n_tokens_mistral': 2056, 'n_tokens_neox': 2039, 'n_words': 486}