<commit_before>import { error } from "../util/util";

const WHITESPACE_RE = /\s/;

const pushChild = (child, stack) => {
  stack[stack.length - 1].children.push(child);
};

const parseOpeningTag = (index, input, length, stack) => {
  let type = "";

  for (; index < length; index++) {
    const char = input[index];

    if (char === ">") {
      const element = {
        type: type,
        children: []
      };

      pushChild(element, stack);
      stack.push(element);

      index += 1;
      break;
    } else if (char === "/" && input[index + 1] === ">") {
      pushChild({
        type: type,
        children: []
      }, stack);

      index += 2;
      break;
    } else {
      type += char;
    }
  }

  return index;
};

const parseClosingTag = (index, input, length, stack) => {
  let type = "";

  for(; index < length; index++) {
    const char = input[index];

    if (char === ">") {
      index += 1;
      break;
    } else {
      type += char;
    }
  }

  const lastElement = stack.pop();
  if (type !== lastElement.type && process.env.MOON_ENV === "development") {
    error(`Unclosed tag "${lastElement.type}", expected "${type}"`);
  }

  return index;
};

const parseText = (index, input, length, stack) => {
  let content = "";

  for (; index < length; index++) {
    const char = input[index];

    if (char === "<") {
      break;
    } else {
      content += char;
    }
  }

  pushChild({
    type: "m-text",
    content: content
  }, stack);

  return index;
};

const parse = (input) => {
  const length = input.length;

  const root = {
    type: "m-fragment",
    children: []
  };

  let stack = [root];

  for (let i = 0; i < length;) {
    const char = input[i];

    if (char === "<") {
      if (input[i + 1] === "!" && input[i + 2] === "-" && input[i + 3] === "-") {
        i = parseComment(i + 4, input, length, stack);
      } else if (input[i + 1] === "/") {
        i = parseClosingTag(i + 2, input, length, stack);<commit_msg>export parser<commit_after>import { error } from "../util/util";

const WHITESPACE_RE = /\s/;

const pushChild = (child, stack) => {
  stack[stack.length - 1].children.push(child);
};

const parseOpeningTag = (index, input, length, stack) => {
  let type = "";

  for (; index < length; index++) {
    const char = input[index];

    if (char === ">") {
      const element = {
        type: type,
        children: []
      };

      pushChild(element, stack);
      stack.push(element);

      index += 1;
      break;
    } else if (char === "/" && input[index + 1] === ">") {
      pushChild({
        type: type,
        children: []
      }, stack);

      index += 2;
      break;
    } else {
      type += char;
    }
  }

  return index;
};

const parseClosingTag = (index, input, length, stack) => {
  let type = "";

  for(; index < length; index++) {
    const char = input[index];

    if (char === ">") {
      index += 1;
      break;
    } else {
      type += char;
    }
  }

  const lastElement = stack.pop();
  if (type !== lastElement.type && process.env.MOON_ENV === "development") {
    error(`Unclosed tag "${lastElement.type}", expected "${type}"`);
  }

  return index;
};

const parseText = (index, input, length, stack) => {
  let content = "";

  for (; index < length; index++) {
    const char = input[index];

    if (char === "<") {
      break;
    } else {
      content += char;
    }
  }

  pushChild({
    type: "m-text",
    content: content
  }, stack);

  return index;
};

export const parse = (input) => {
  const length = input.length;

  const root = {
    type: "m-fragment",
    children: []
  };

  let stack = [root];

  for (let i = 0; i < length;) {
    const char = input[i];

    if (char === "<") {
      if (input[i + 1] === "!" && input[i + 2] === "-" && input[i + 3] === "-") {
        i = parseComment(i + 4, input, length, stack);
      } else if (input[i + 1] === "/") {
        i = parseClosingTag(i + 2, input, length, stack);
{'fraction_non_alphanumeric': 0.1954167715940569, 'fraction_numerical': 0.0060438176781667085, 'mean_word_length': 2.1699920191540305, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7450084', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1316, 'n_words': 349}