<commit_before>/**
 * A simplifier implemented in TypeScript (thus internal to Salve).
 *
 * @author <NAME>
 * @license MPL 2.0
 * @copyright Mangalam Research Center for Buddhist Languages
 */
import * as sax from "sax";

import { registry, ValueError, ValueValidationError } from "../../datatypes";
import { readTreeFromJSON } from "../../json-format/read";
import { AnyName, ConcreteName, Data, Grammar, Name, NameChoice, NsName,
         Value } from "../../patterns";
import * as relaxng from "../../schemas/relaxng";
import { BasicParser, Element, Text, Validator } from "../parser";
import { registerSimplifier, SchemaSimplifierOptions,
         SimplificationResult } from "../schema-simplification";
import { SchemaValidationError } from "../schema-validation";
import * as simplifier from "../simplifier";
import { findMultiNames, getName, indexBy } from "../simplifier/util";
import { BaseSimplifier } from "./base";
import { fromQNameToURI, localName } from "./common";

function makeNamePattern(el: Element): ConcreteName {
  const first = el.children[0] as Element;
  const second = el.children[1] as Element;
  switch (el.local) {
    case "name":
      return new Name("", el.mustGetAttribute("ns"), el.text);
    case "choice":
      return new NameChoice("", makeNamePattern(first),
                            makeNamePattern(second));
    case "anyName": {
      return new AnyName("",
                         first !== undefined ? makeNamePattern(first) :
                         undefined);
    }
    case "nsName": {
      return new NsName("",
                        el.mustGetAttribute("ns"),
                        first !== undefined ? makeNamePattern(first) :
                        undefined);
    }
    case "except":
      return makeNamePattern(first);
    default:
      throw new Error(`unexpected element in name pattern ${el.local}`);
  }
}

enum ContentType {
  EMPTY,
  COMPLEX,
  SIMPLE,
}

function groupable(a: ContentType, b: ContentType): boolean {
  return a === ContentType.EMPTY || b === ContentType.EMPTY ||
    (a === ContentType.COMPLEX && b === ContentType.COMPLEX);
}

function computeContentType(pattern: Element): ContentType | null {
  const name = pattern.local;
  switch (name) {
    case "value":
    case "data":
    case "list":
      return ContentType.SIMPLE;
    case "text":
    case "ref":
      return ContentType.COMPLEX;
    case "empty":
    case "attribute":
      return ContentType.EMPTY;
    case "interleave":
    case "group": {
      const firstCt = computeContentType(pattern.children[0] as Element);

      if (firstCt === null) {
        return null;
      }

      const secondCt = computeContentType(pattern.children[1] as Element);

      return secondCt !== null && groupable(firstCt, secondCt) ?
        (firstCt > secondCt ? firstCt : secondCt) : null;
    }
    case "oneOrMore":
      const ct = computeContentType(pattern.children[0] as Element);

      // The test would be
      //
      // ct !== null && groupable(ct, ct)
      //
      // but the only thing not groupable with itself is ContentType.SIMPLE and
      // if ct === null then forcibly ct !== ContentType.SIMPLE is true so we
      // can simplify to the following.
      return ct !== ContentType.SIMPLE ? ct : null;
    case "choice": {
      const firstCt = computeContentType(pattern.children[0] as Element);

      if (firstCt === null) {
        return null;
      }

      const secondCt = computeContentType(pattern.children[1] as Element);

      return secondCt !== null ?
        (firstCt > secondCt ? firstCt : secondCt) : null;
    }
    default:
      throw new Error(`unexpected element: ${name}`);
  }
}

interface State {
  inStart: boolean;
  inAttribute: boolean;<commit_msg>Optimize contentType computation.<commit_after>/**
 * A simplifier implemented in TypeScript (thus internal to Salve).
 *
 * @author <NAME>
 * @license MPL 2.0
 * @copyright Mangalam Research Center for Buddhist Languages
 */
import * as sax from "sax";

import { registry, ValueError, ValueValidationError } from "../../datatypes";
import { readTreeFromJSON } from "../../json-format/read";
import { AnyName, ConcreteName, Data, Grammar, Name, NameChoice, NsName,
         Value } from "../../patterns";
import * as relaxng from "../../schemas/relaxng";
import { BasicParser, Element, Text, Validator } from "../parser";
import { registerSimplifier, SchemaSimplifierOptions,
         SimplificationResult } from "../schema-simplification";
import { SchemaValidationError } from "../schema-validation";
import * as simplifier from "../simplifier";
import { findMultiNames, getName, indexBy } from "../simplifier/util";
import { BaseSimplifier } from "./base";
import { fromQNameToURI, localName } from "./common";

function makeNamePattern(el: Element): ConcreteName {
  const first = el.children[0] as Element;
  const second = el.children[1] as Element;
  switch (el.local) {
    case "name":
      return new Name("", el.mustGetAttribute("ns"), el.text);
    case "choice":
      return new NameChoice("", makeNamePattern(first),
                            makeNamePattern(second));
    case "anyName": {
      return new AnyName("",
                         first !== undefined ? makeNamePattern(first) :
                         undefined);
    }
    case "nsName": {
      return new NsName("",
                        el.mustGetAttribute("ns"),
                        first !== undefined ? makeNamePattern(first) :
                        undefined);
    }
    case "except":
      return makeNamePattern(first);
    default:
      throw new Error(`unexpected element in name pattern ${el.local}`);
  }
}

const enum ContentType {
  EMPTY,
  COMPLEX,
  SIMPLE,
}

function computeContentType(pattern: Element): ContentType | null {
  const name = pattern.local;
  switch (name) {
    case "group":
    case "interleave": {
      const firstCt = computeContentType(pattern.children[0] as Element);
      if (firstCt === null) {
        return null;
      }

      const secondCt = computeContentType(pattern.children[1] as Element);
      if (secondCt === null) {
        return null;
      }

      // These tests combine the groupable(firstCt, secondCt) test together with
      // the requirement that we return the content type which is the greatest.
      if (firstCt === ContentType.COMPLEX && secondCt === ContentType.COMPLEX) {
        return ContentType.COMPLEX;
      }

      if (firstCt === ContentType.EMPTY) {
        return secondCt;
      }

      return (secondCt === ContentType.EMPTY) ? firstCt : null;
    }
    case "choice": {
      // We check secondCt first because the schema simplification puts
      // ``empty`` in the first slot. If the first slot is ``empty``, there's no
      // opportunity for short-circuiting the computation. On the other hand if
      // the second child has a simple content type, we don't neet to know
      // whether the first child is empty or not.
      const secondCt = computeContentType(pattern.children[1] as Element);

      // If the secondCt is simple, we already know what the max value of the
      // two content types is and we can return right away.
      if (secondCt === null || secondCt === ContentType.SIMPLE) {
        return secondCt;
      }

      const firstCt = computeContentType(pattern.children[0] as Element);

      return firstCt !== null ?
        (firstCt > secondCt ? firstCt : secondCt) : null;
    }
    case "oneOrMore":
      const ct = computeContentType(pattern.children[0] as Element);

      // The test would be
      //
      // ct !== null && groupable(ct, ct)
      //
      // but the only thing not groupable with itself is ContentType.SIMPLE and
      // if ct === null then forcibly ct !== ContentType.SIMPLE is true so we
      // can simplify to the following.
      return ct !== ContentType.SIMPLE ? ct : null;
    case "text":
    case "ref":
      return ContentType.COMPLEX;
    case "empty":
    case "attribute":
      return ContentType.EMPTY;
    case "value":
    case "data":
    case "list":
      return ContentType.SIMPLE;
    default:
      throw new Error(`unexpected element: ${name}`);
  }
}

interface State {
  inStart: boolean;
  inAttribute: boolean;
{'fraction_non_alphanumeric': 0.12134449702705982, 'fraction_numerical': 0.002184200946487077, 'mean_word_length': 2.6228571428571428, 'pattern_counts': {'":': 32, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6737867', 'n_tokens_mistral': 2371, 'n_tokens_neox': 2265, 'n_words': 729}