<commit_before>(function(exports) {
  'use strict';

  var svgAttrTokenNames = ["attributename", "attributetype", "basefrequency", "baseprofile", "calcmode", "clippathunits", "contentscripttype", "contentstyletype", "diffuseconstant", "edgemode", "externalresourcesrequired", "filterres", "filterunits", "glyphref", "gradienttransform", "gradientunits", "kernelmatrix", "kernelunitlength", "keypoints", "keysplines", "keytimes", "lengthadjust", "limitingconeangle", "markerheight", "markerunits", "markerwidth", "maskcontentunits", "maskunits", "numoctaves", "pathlength", "patterncontentunits", "patterntransform", "patternunits", "pointsatx", "pointsaty", "pointsatz", "preservealpha", "preserveaspectratio", "primitiveunits", "refx", "refy", "repeatcount", "repeatdur", "requiredextensions", "requiredfeatures", "specularconstant", "specularexponent", "spreadmethod", "startoffset", "stddeviation", "stitchtiles", "surfacescale", "systemlanguage", "tablevalues", "targetx", "targety", "textlength", "viewbox", "viewtarget", "xchannelselector", "ychannelselector", "zoomandpan"];
  var svgAttrElementNames = ["attributeName", "attributeType", "baseFrequency", "baseProfile", "calcMode", "clipPathUnits", "contentScriptType", "contentStyleType", "diffuseConstant", "edgeMode", "externalResourcesRequired", "filterRes", "filterUnits", "glyphRef", "gradientTransform", "gradientUnits", "kernelMatrix", "kernelUnitLength", "keyPoints", "keySplines", "keyTimes", "lengthAdjust", "limitingConeAngle", "markerHeight", "markerUnits", "markerWidth", "maskContentUnits", "maskUnits", "numOctaves", "pathLength", "patternContentUnits", "patternTransform", "patternUnits", "pointsAtX", "pointsAtY", "pointsAtZ", "preserveAlpha", "preserveAspectRatio", "primitiveUnits", "refX", "refY", "repeatCount", "repeatDur", "requiredExtensions", "requiredFeatures", "specularConstant", "specularExponent", "spreadMethod", "startOffset", "stdDeviation", "stitchTiles", "surfaceScale", "systemLanguage", "tableValues", "targetX", "targetY", "textLength", "viewBox", "viewTarget", "xChannelSelector", "yChannelSelector", "zoomAndPan"];
  var svgCaseFix = {
    altglyph:	"altGlyph",
    altglyphdef:	"altGlyphDef",
    altglyphitem:	"altGlyphItem",
    animatecolor:	"animateColor",
    animatemotion:	"animateMotion",
    animatetransform:	"animateTransform",
    clippath:	"clipPath",
    feblend:	"feBlend",
    fecolormatrix:	"feColorMatrix",
    fecomponenttransfer:	"feComponentTransfer",
    fecomposite:	"feComposite",
    feconvolvematrix:	"feConvolveMatrix",
    fediffuselighting:	"feDiffuseLighting",
    fedisplacementmap:	"feDisplacementMap",
    fedistantlight:	"feDistantLight",
    feflood:	"feFlood",
    fefunca:	"feFuncA",
    fefuncb:	"feFuncB",
    fefuncg:	"feFuncG",
    fefuncr:	"feFuncR",
    fegaussianblur:	"feGaussianBlur",
    feimage:	"feImage",
    femerge:	"feMerge",
    femergenode:	"feMergeNode",
    femorphology:	"feMorphology",
    feoffset:	"feOffset",
    fepointlight:	"fePointLight",
    fespecularlighting:	"feSpecularLighting",
    fespotlight:	"feSpotLight",
    fetile:	"feTile",
    feturbulence:	"feTurbulence",
    foreignobject:	"foreignObject",
    glyphref:	"glyphRef",
    lineargradient:	"linearGradient",
    radialgradient:	"radialGradient",
    textpath:	"textPath"
  };

  var common = require("./common.js");
  var namespace = common.namespace;
  var characters = common.characters;
  var tokenType = common.tokenType;
  var nodeType = common.nodeType;
  var Tokenizer = require("./tokenizer.js").Tokenizer;
  var Constructor = require("./constructor.js").Constructor;
  var spaceCharacters = [characters.SPACE, characters.TAB, characters.LF, characters.FF, characters.CR];
  var invalidCharacters = common.invalidCharacters;

  function remapNewlines(chr) {
    return chr.charCodeAt(0) === characters.CR ? String.fromCharCode(characters.LF) : chr;
  }

  function notSpaceCharacterToken(token) {
    return spaceCharacters.indexOf(token[1].charCodeAt(0)) === -1;
  }


  function get_tagName(node) {
    return (typeof(node) === "object") ? node._tagName : node;
  }

  function Parser(html, options) {
    this.options = options || {};
    this.mode = this.modes.initial;
    this.templateInsertionModes = [];
    this.originalMode = null;
    this.headElementPointer = null;
    this.formElementPointer = null;
    this.openElements = [];
    this.lastPosition = null;
    this.scriptingFlag = true;
    this.fosterParented = false;
    this.activeFormattingElements = [];
    this.pendingTableCharacterTokens =  null;
    this.ignoreToken = null;
    this.paused = true;

    this.input = html.split("").filter(function(chr, index, input) {
      return !(input[index+1] && input[index+1].charCodeAt(0) === characters.LF && chr.charCodeAt(0) === characters.CR);
    }).map(remapNewlines);

    this.tokenizer = new Tokenizer(this.options, this);
    this.constructor = (options && options.document) ? options.document : new Constructor();
    this.tokens = [];
    this.fragmentCase = this.options.fragment ? true : false;

    switch(this.options.fragment) {
      case "textarea":
        this.tokenizer.setState(this.tokenizer.states.RCDATA);
        this.mode = this.modes.text;
        this.originalMode = this.modes.in_body;
        break;
      case "plaintext":
        this.mode = this.modes.in_body;
        this.tokenizer.setState(this.tokenizer.states.plaintext);
        break;
      case "head": case "body": this.mode = this.modes.in_body; break;
      case "html": this.mode = this.modes.before_head; break;
      case "div": this.mode = this.modes.in_body; break;
      case "style":
        this.tokenizer.state =  this.tokenizer.states.rawtext;
        this.originalMode = this.modes.in_body;
        this.mode = this.modes.text;
        break;
      case "frameset": this.mode = this.modes.in_frameset; break;
      case "table": this.mode = this.modes.in_table; break;
      case "tbody": this.mode = this.modes.in_table_body; break;
      case "caption":
        this.mode = this.modes.in_caption;
        this.constructor._tagName = "caption";
        break;
      case "colgroup": this.mode = this.modes.in_column_group; break;
      case "tr":
        this.mode = this.modes.in_row;
        this.constructor._tagName = "tr";
        break;
      case "td": this.mode = this.modes.in_cell; break;
      case "select": this.mode = this.modes.in_select; break;
      case undefined: break;
      default: console.log('Undefined fragment', this.options.fragment);
    }
  }

  Parser.prototype.run = function() {
    this.paused = false;
    var addToken = this.addToken.bind(this),
      result;
    do {
      if (this.paused) {
        return;
      }

      if (this.invalidCharacter(this.input)) {
        this.tokens.push(tokenType.parseError);
      }
      this.lastPosition = this.tokenizer.position;
      result = this.tokenizer.parse(this.input);
      if (Array.isArray(result)) {
        result.forEach(addToken);
      }
    } while(result !== false);

    if (this.options.type === "tree") {
      this.addToken([tokenType.EOF]);
    }

    if (this.options.complete) {
      this.options.complete.call(this.options.document);
    }
  };

  Parser.prototype.pause = function() {
    this.paused = true;
  };

  Parser.prototype.integrationPoint = {
    MathMLtext: 1,<commit_msg>Remove complete callback on first call<commit_after>(function(exports) {
  'use strict';

  var svgAttrTokenNames = ["attributename", "attributetype", "basefrequency", "baseprofile", "calcmode", "clippathunits", "contentscripttype", "contentstyletype", "diffuseconstant", "edgemode", "externalresourcesrequired", "filterres", "filterunits", "glyphref", "gradienttransform", "gradientunits", "kernelmatrix", "kernelunitlength", "keypoints", "keysplines", "keytimes", "lengthadjust", "limitingconeangle", "markerheight", "markerunits", "markerwidth", "maskcontentunits", "maskunits", "numoctaves", "pathlength", "patterncontentunits", "patterntransform", "patternunits", "pointsatx", "pointsaty", "pointsatz", "preservealpha", "preserveaspectratio", "primitiveunits", "refx", "refy", "repeatcount", "repeatdur", "requiredextensions", "requiredfeatures", "specularconstant", "specularexponent", "spreadmethod", "startoffset", "stddeviation", "stitchtiles", "surfacescale", "systemlanguage", "tablevalues", "targetx", "targety", "textlength", "viewbox", "viewtarget", "xchannelselector", "ychannelselector", "zoomandpan"];
  var svgAttrElementNames = ["attributeName", "attributeType", "baseFrequency", "baseProfile", "calcMode", "clipPathUnits", "contentScriptType", "contentStyleType", "diffuseConstant", "edgeMode", "externalResourcesRequired", "filterRes", "filterUnits", "glyphRef", "gradientTransform", "gradientUnits", "kernelMatrix", "kernelUnitLength", "keyPoints", "keySplines", "keyTimes", "lengthAdjust", "limitingConeAngle", "markerHeight", "markerUnits", "markerWidth", "maskContentUnits", "maskUnits", "numOctaves", "pathLength", "patternContentUnits", "patternTransform", "patternUnits", "pointsAtX", "pointsAtY", "pointsAtZ", "preserveAlpha", "preserveAspectRatio", "primitiveUnits", "refX", "refY", "repeatCount", "repeatDur", "requiredExtensions", "requiredFeatures", "specularConstant", "specularExponent", "spreadMethod", "startOffset", "stdDeviation", "stitchTiles", "surfaceScale", "systemLanguage", "tableValues", "targetX", "targetY", "textLength", "viewBox", "viewTarget", "xChannelSelector", "yChannelSelector", "zoomAndPan"];
  var svgCaseFix = {
    altglyph:	"altGlyph",
    altglyphdef:	"altGlyphDef",
    altglyphitem:	"altGlyphItem",
    animatecolor:	"animateColor",
    animatemotion:	"animateMotion",
    animatetransform:	"animateTransform",
    clippath:	"clipPath",
    feblend:	"feBlend",
    fecolormatrix:	"feColorMatrix",
    fecomponenttransfer:	"feComponentTransfer",
    fecomposite:	"feComposite",
    feconvolvematrix:	"feConvolveMatrix",
    fediffuselighting:	"feDiffuseLighting",
    fedisplacementmap:	"feDisplacementMap",
    fedistantlight:	"feDistantLight",
    feflood:	"feFlood",
    fefunca:	"feFuncA",
    fefuncb:	"feFuncB",
    fefuncg:	"feFuncG",
    fefuncr:	"feFuncR",
    fegaussianblur:	"feGaussianBlur",
    feimage:	"feImage",
    femerge:	"feMerge",
    femergenode:	"feMergeNode",
    femorphology:	"feMorphology",
    feoffset:	"feOffset",
    fepointlight:	"fePointLight",
    fespecularlighting:	"feSpecularLighting",
    fespotlight:	"feSpotLight",
    fetile:	"feTile",
    feturbulence:	"feTurbulence",
    foreignobject:	"foreignObject",
    glyphref:	"glyphRef",
    lineargradient:	"linearGradient",
    radialgradient:	"radialGradient",
    textpath:	"textPath"
  };

  var common = require("./common.js");
  var namespace = common.namespace;
  var characters = common.characters;
  var tokenType = common.tokenType;
  var nodeType = common.nodeType;
  var Tokenizer = require("./tokenizer.js").Tokenizer;
  var Constructor = require("./constructor.js").Constructor;
  var spaceCharacters = [characters.SPACE, characters.TAB, characters.LF, characters.FF, characters.CR];
  var invalidCharacters = common.invalidCharacters;

  function remapNewlines(chr) {
    return chr.charCodeAt(0) === characters.CR ? String.fromCharCode(characters.LF) : chr;
  }

  function notSpaceCharacterToken(token) {
    return spaceCharacters.indexOf(token[1].charCodeAt(0)) === -1;
  }


  function get_tagName(node) {
    return (typeof(node) === "object") ? node._tagName : node;
  }

  function Parser(html, options) {
    this.options = options || {};
    this.mode = this.modes.initial;
    this.templateInsertionModes = [];
    this.originalMode = null;
    this.headElementPointer = null;
    this.formElementPointer = null;
    this.openElements = [];
    this.lastPosition = null;
    this.scriptingFlag = true;
    this.fosterParented = false;
    this.activeFormattingElements = [];
    this.pendingTableCharacterTokens =  null;
    this.ignoreToken = null;
    this.paused = true;

    this.input = html.split("").filter(function(chr, index, input) {
      return !(input[index+1] && input[index+1].charCodeAt(0) === characters.LF && chr.charCodeAt(0) === characters.CR);
    }).map(remapNewlines);

    this.tokenizer = new Tokenizer(this.options, this);
    this.constructor = (options && options.document) ? options.document : new Constructor();
    this.tokens = [];
    this.fragmentCase = this.options.fragment ? true : false;

    switch(this.options.fragment) {
      case "textarea":
        this.tokenizer.setState(this.tokenizer.states.RCDATA);
        this.mode = this.modes.text;
        this.originalMode = this.modes.in_body;
        break;
      case "plaintext":
        this.mode = this.modes.in_body;
        this.tokenizer.setState(this.tokenizer.states.plaintext);
        break;
      case "head": case "body": this.mode = this.modes.in_body; break;
      case "html": this.mode = this.modes.before_head; break;
      case "div": this.mode = this.modes.in_body; break;
      case "style":
        this.tokenizer.state =  this.tokenizer.states.rawtext;
        this.originalMode = this.modes.in_body;
        this.mode = this.modes.text;
        break;
      case "frameset": this.mode = this.modes.in_frameset; break;
      case "table": this.mode = this.modes.in_table; break;
      case "tbody": this.mode = this.modes.in_table_body; break;
      case "caption":
        this.mode = this.modes.in_caption;
        this.constructor._tagName = "caption";
        break;
      case "colgroup": this.mode = this.modes.in_column_group; break;
      case "tr":
        this.mode = this.modes.in_row;
        this.constructor._tagName = "tr";
        break;
      case "td": this.mode = this.modes.in_cell; break;
      case "select": this.mode = this.modes.in_select; break;
      case undefined: break;
      default: console.log('Undefined fragment', this.options.fragment);
    }
  }

  Parser.prototype.run = function() {
    this.paused = false;
    var addToken = this.addToken.bind(this),
      result;
    do {
      if (this.paused) {
        return;
      }

      if (this.invalidCharacter(this.input)) {
        this.tokens.push(tokenType.parseError);
      }
      this.lastPosition = this.tokenizer.position;
      result = this.tokenizer.parse(this.input);
      if (Array.isArray(result)) {
        result.forEach(addToken);
      }
    } while(result !== false);

    if (this.options.type === "tree") {
      this.addToken([tokenType.EOF]);
    }

    if (this.options.complete) {
      this.options.complete.call(this.options.document);
      this.options.complete = null;
    }
  };

  Parser.prototype.pause = function() {
    this.paused = true;
  };

  Parser.prototype.integrationPoint = {
    MathMLtext: 1,
{'fraction_non_alphanumeric': 0.14820328889490425, 'fraction_numerical': 0.0012181092237937335, 'mean_word_length': 4.231150442477876, 'pattern_counts': {'":': 30, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5657906', 'n_tokens_mistral': 4782, 'n_tokens_neox': 4619, 'n_words': 884}