<commit_before>var code = '' + function outside () {
  var out;

  function insideUno () {
    var uno;
    function nestedInsideUno () {
      var nest;
    }
  }

  function insideDos () {
    var dos;
  }
};

// should sectionize as follows
// { 1-3   :  FunctionDeclaration outside
//   4-5   :  FunctionDeclaration insideUno
//   6-8   :  FunctionDeclaration nestedInsideUno
//   9-9   :  FunctionDeclaration insideUno
//   10-10 :  FunctionDeclaration outside
//   11-13 :  FunctionDeclaration insideDos
//   14-14 :  FunctionDeclaration outside
// }
// Alternatively as follows (smallest section to largest)
//
// { 6-8   :  nestedInsideUno
//   4-9   :  insideUno
//   11-13 :  insideDos
//   1-14  :  outside
// }
// 
// Or sorted by starting line.
// This could be the prep to convert to an array lineIndex -> BlockStatement
// { 1-14  :  outside
//   4-9   :  insideUno
//   6-8   :  nestedInsideUno
//   11-13 :  insideDos
// }
// [1] - outside
// [2] - outside
// [3] - outside
// [4] - inside
// etc...

var esprima  =  require('esprima');
var util     =  require('util');
var ast      =  esprima.parse(code, { loc: true });
var toString =  Object.prototype.toString;

function inspect (o, depth) {
  console.log(util.inspect(o, false, depth || 5, true));
}

function isObject (obj) {
  return toString.call(obj) === '[object Object]';
}

function flatten(acc, node, type) {

  if (Array.isArray(node)) {
    return node.forEach(function (n) {
      flatten(acc, n, type);  
    });
  }

  if (!isObject(node)) return;

  Object.keys(node).forEach(function (k) {
    var child = node[k];
    if (child && child.type === type) { 
      child.parent = node;
      acc.push(child);
    }
  });
  Object.keys(node)
    .filter(function (k) { return k !== 'parent'; })
    .forEach(function (k) {
      flatten(acc, node[k], type);
    });
}

var acc = [];
flatten(acc, ast, 'BlockStatement');
console.log(acc);






























<commit_msg>line array with pointers to appropriate block<commit_after>var code = '' + function outside () {
  var out;

  function insideUno () {
    var uno;
    function nestedInsideUno () {
      var nest;
    }
  }

  function insideDos () {
    var dos;
  }
};

// should sectionize as follows
// { 1-3   :  FunctionDeclaration outside
//   4-5   :  FunctionDeclaration insideUno
//   6-8   :  FunctionDeclaration nestedInsideUno
//   9-9   :  FunctionDeclaration insideUno
//   10-10 :  FunctionDeclaration outside
//   11-13 :  FunctionDeclaration insideDos
//   14-14 :  FunctionDeclaration outside
// }
// Alternatively as follows (smallest section to largest)
//
// { 6-8   :  nestedInsideUno
//   4-9   :  insideUno
//   11-13 :  insideDos
//   1-14  :  outside
// }
// 
// Or sorted by starting line.
// This could be the prep to convert to an array lineIndex -> BlockStatement
// { 1-14  :  outside
//   4-9   :  insideUno
//   6-8   :  nestedInsideUno
//   11-13 :  insideDos
// }
// [1] - outside
// [2] - outside
// [3] - outside
// [4] - inside
// etc...

'use strict';
var esprima  =  require('esprima');
var util     =  require('util');
var ast      =  esprima.parse(code, { loc: true });
var toString =  Object.prototype.toString;

function inspect (o, depth) {
  console.log(util.inspect(o, false, depth || 5, true));
}

function isObject (obj) {
  return toString.call(obj) === '[object Object]';
}

function flatten(acc, node, type) {
  // flattens ast breadth first which results in array of matching nodes sorted by location

  if (Array.isArray(node)) {
    return node.forEach(function (n) {
      flatten(acc, n, type);  
    });
  }

  if (!isObject(node)) return;

  Object.keys(node).forEach(function (k) {
    var child = node[k];
    if (child && child.type === type) { 
      child.parent = node;
      acc.push(child);
    }
  });

  Object.keys(node)
    .filter(function (k) { return k !== 'parent'; })
    .forEach(function (k) {
      flatten(acc, node[k], type);
    });
  return acc;
}

function statementLines (flattenedAst) {
  var lines = [];
  // since the flattened ast is breadth first and outer statement to inner, we can first fill
  // the entire range with the outer block and let inner blocks override where appropriate
  flattenedAst.forEach(function (statement) {
    var loc = statement.loc;
    for (var lineno = loc.start.line; lineno <= loc.end.line; lineno++) {
      lines[lineno] = statement;
    }    
  });

  return lines;
}

var flattenedAst = flatten([], ast, 'BlockStatement');

var lines = statementLines(flattenedAst);

inspect(lines);
lines.forEach(function (line) {
  console.log(line.parent.id.name);
});


























{'fraction_non_alphanumeric': 0.14718521701761925, 'fraction_numerical': 0.02019767941555651, 'mean_word_length': 2.4973703981968445, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5922360', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1532, 'n_words': 413}