<commit_before>
//TODO: AST Manipulation API.
(function (){
"use strict";

var utils = require("./utils.js");
var ENUM =
{
 ROOT : "ROOT",
 P : "P",
 BLOCKQUOTE : "BLOCKQUOTE",
 COMMENT : "COMMENT",
 PRE : "PRE",
 DIV : "DIV",
 ID : "ID",
 CLASS : "CLASS",
 UL_LI : "UL_LI",
 OL_LI : "OL_LI",
 LI : "LI", //After Transform
 UL : "UL", //After Transform
 OL : "OL", //After Transform
 HEADER : "HEADER",
 DT : "DT",
 DD : "DD",
 DL : "DL", //After Transform
 TH : "TH",
 TD : "TD",
 HR : "HR",
 TR : "TR", //After Transform
 TRSEP : "TRSEP",
 TABLE : "TABLE", //After Transform
 LINK_INT : "LINK_INT",
 LINK_EXT : "LINK_EXT",
 LINK_IMG : "LINK_IMG",
 LINK_WIKI : "LINK_WIKI",
 DEL : "DEL",
 INS : "INS",
 UNDER : "UNDER",
 SUB : "SUB",
 SUP : "SUP",
 EM : "EM",
 STRONG : "STRONG",
 CODE : "CODE",
 TEXT : "TEXT" //After Transform
};


function ASTNode(type, attr)
{
 this.type = type || "";
 this.nodes = [];
}
function create(type, attr)
{
 return new ASTNode(type, attr);
}

ASTNode.create = create;
ASTNode.types = ENUM;
ASTNode.prototype = (function (){

 var switchAppend =
 {
  TRSEP : appendTable,
  TH : appendTable,
  TD : appendTable,
  DT : appendDL,
  DD : appendDL,
  UL_LI : appendUL,
  OL_LI : appendULOL,
  LI : appendULOL,
 },

 /*
 Private Methods : Append
 ------------------------
 */
 function appendTable(node)
 {
  var last = this.last(),
   isCell = node.type === enumAST.TD || node.type === enumAST.TH,
   isRow = node.type === enumAST.TRSEP;

  if (!(last && last.type === enumAST.TABLE))
  {
   if (isRow) {return;}
   last = ASTNode.create(enumAST.TABLE);
   appendNode.call(this, last);
  }
  if (last.nodes.length <= 0)
  {
   appendNode.call(last, ASTNode.create(enumAST.TR));
  }

  if (isCell)
  {
   appendNode.call(last.last(), node);
  }
  else if (isRow)
  {
   node.type = enumAST.TR;
   appendNode.call(last, node);
  }
 }
 
 function appendDL(node)
 {
  var last = this.last();
  if (!(last && last.type === enumAST.DL))
  {
   last = ASTNode.create(enumAST.DL);
   appendNode.call(this, last);
  }
  appendNode.call(last, node);
 }
 
 function appendULOL(node)
 {
  var listType = node.type === enumAST.OL_LI ? enumAST.OL : enumAST.UL,
   last = this.last();

  if (!(last && last.type === listType))
  {
   last = ASTEnum.create(listType);
   appendNode.call(this, last);
  }
  appendNode.call(last, node);
  node.type = enumAST.LI;
 }
 
 function appendText(text)
 {
  var textNode = ASTNode.create(ENUM.TEXT);
  textNode.nodes.push(text);
  this.nodes.push(textNode);
 }
 
 function appendNode(node)
 {
  if (!utils.hasOwn(ENUM, node.type))
  {
   throw TypeError(node.type + " is not member of ASTNode.types.");
  }
  this.nodes.push(node);
 }

 /*
 Public Methods
 --------------
 */
 
 function append(nodeText)
 {
  var isNode = nodeText instanceof ASTNode,
   nodeFunc = isNode ? switchAppend[nodeText.type] : null;
 
  if (utils.isString(nodeText))
  {
   appendText.call(this, nodeText);
  }
  else if (nodeFunc instanceof Function)
  {
   nodeFunc.call(this, nodeText);
  }
  else if (isNode)
  {
   appendNode.call(this, nodeText);
  }
  return this;
 }
 
 function first()
 {
  return this.nodes[0];
 }

 function last()
 {
  return this.nodes[this.nodes.length - 1];
 }
 



 return {
  append : append,
  first : first,
  last : last
 };

}());


if (typeof module === "object" && module.exports)
{
 module.exports = ASTNode;
}
}());
<commit_msg>ULOL Typo Fix<commit_after>
//TODO: AST Manipulation API.
(function (){
"use strict";

var utils = require("./utils.js");
var ENUM =
{
 ROOT : "ROOT",
 P : "P",
 BLOCKQUOTE : "BLOCKQUOTE",
 COMMENT : "COMMENT",
 PRE : "PRE",
 DIV : "DIV",
 ID : "ID",
 CLASS : "CLASS",
 UL_LI : "UL_LI",
 OL_LI : "OL_LI",
 LI : "LI", //After Transform
 UL : "UL", //After Transform
 OL : "OL", //After Transform
 HEADER : "HEADER",
 DT : "DT",
 DD : "DD",
 DL : "DL", //After Transform
 TH : "TH",
 TD : "TD",
 HR : "HR",
 TR : "TR", //After Transform
 TRSEP : "TRSEP",
 TABLE : "TABLE", //After Transform
 LINK_INT : "LINK_INT",
 LINK_EXT : "LINK_EXT",
 LINK_IMG : "LINK_IMG",
 LINK_WIKI : "LINK_WIKI",
 DEL : "DEL",
 INS : "INS",
 UNDER : "UNDER",
 SUB : "SUB",
 SUP : "SUP",
 EM : "EM",
 STRONG : "STRONG",
 CODE : "CODE",
 TEXT : "TEXT" //After Transform
};


function ASTNode(type, attr)
{
 this.type = type || "";
 this.nodes = [];
}
function create(type, attr)
{
 return new ASTNode(type, attr);
}

ASTNode.create = create;
ASTNode.types = ENUM;
ASTNode.prototype = (function (){

 var switchAppend =
 {
  TRSEP : appendTable,
  TH : appendTable,
  TD : appendTable,
  DT : appendDL,
  DD : appendDL,
  UL_LI : appendULOL,
  OL_LI : appendULOL,
  LI : appendULOL,
 },

 /*
 Private Methods : Append
 ------------------------
 */
 function appendTable(node)
 {
  var last = this.last(),
   isCell = node.type === enumAST.TD || node.type === enumAST.TH,
   isRow = node.type === enumAST.TRSEP;

  if (!(last && last.type === enumAST.TABLE))
  {
   if (isRow) {return;}
   last = ASTNode.create(enumAST.TABLE);
   appendNode.call(this, last);
  }
  if (last.nodes.length <= 0)
  {
   appendNode.call(last, ASTNode.create(enumAST.TR));
  }

  if (isCell)
  {
   appendNode.call(last.last(), node);
  }
  else if (isRow)
  {
   node.type = enumAST.TR;
   appendNode.call(last, node);
  }
 }
 
 function appendDL(node)
 {
  var last = this.last();
  if (!(last && last.type === enumAST.DL))
  {
   last = ASTNode.create(enumAST.DL);
   appendNode.call(this, last);
  }
  appendNode.call(last, node);
 }
 
 function appendULOL(node)
 {
  var listType = node.type === enumAST.OL_LI ? enumAST.OL : enumAST.UL,
   last = this.last();
   
  if (!(last && last.type === listType))
  {
   last = ASTEnum.create(listType);
   appendNode.call(this, last);
  }
  appendNode.call(last, node);
  node.type = enumAST.LI;
 }
 
 function appendText(text)
 {
  var textNode = ASTNode.create(ENUM.TEXT);
  textNode.nodes.push(text);
  this.nodes.push(textNode);
 }
 
 function appendNode(node)
 {
  if (!utils.hasOwn(ENUM, node.type))
  {
   throw TypeError(node.type + " is not member of ASTNode.types");
  }
  this.nodes.push(node);
 }

 /*
 Public Methods
 --------------
 */
 
 function append(nodeText)
 {
  var isNode = nodeText instanceof ASTNode,
   nodeFunc = isNode ? switchAppend[nodeText.type] : null;
 
  if (utils.isString(nodeText))
  {
   appendText.call(this, nodeText);
  }
  else if (nodeFunc instanceof Function)
  {
   nodeFunc.call(this, nodeText);
  }
  else if (isNode)
  {
   appendNode.call(this, nodeText);
  }
  return this;
 }
 
 function first()
 {
  return this.nodes[0];
 }

 function last()
 {
  return this.nodes[this.nodes.length - 1];
 }
 



 return {
  append : append,
  first : first,
  last : last
 };

}());


if (typeof module === "object" && module.exports)
{
 module.exports = ASTNode;
}
}());

{'fraction_non_alphanumeric': 0.18375440658049352, 'fraction_numerical': 0.0008813160987074031, 'mean_word_length': 2.984201287302516, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '863144', 'n_tokens_mistral': 2645, 'n_tokens_neox': 2517, 'n_words': 582}