<commit_before>/** @jsx createElement */
import _ from 'lodash'
import {createElement, Phrase} from 'lacona-phrase'
import parse from '../parse'
import {reconcile} from '../reconcile'

function addSeparator (child, separator) {
  if (child.props && child.props.optional) {
    const newChild = _.merge({}, child, {props: {optional: false}})

    return <Sequence optional={true} merge={true}>{newChild}{separator}</Sequence>
  } else {
    return <Sequence merge={true}>{child}{separator}</Sequence>
  }
}

export default class Sequence extends Phrase {
  describe() {
    let content, separator
    if (this.props.children[0] && this.props.children[0].Constructor === 'content') {
      content = this.props.children[0].children
      if (this.props.children[1] && this.props.children[1].Constructor === 'separator') {
        separator = this.props.children[1].children[0]
      }
    } else {
      return //no content, we're good to go!
    }

    if (separator) {
      return (
        <sequence {...this.props}>
          {_.chain(content.slice(0, -1))
            .map(_.partial(addSeparator, _, separator))
            .concat(_.last(content))
            .value()
          }
        </sequence>
      )
    } else {
      return <sequence {...this.props}>{content}</sequence>
    }
  }

  *_handleParse(input, options) {
    this.childPhrases = reconcile({descriptor: this.props.children, phrase: this.childPhrases, options})

    yield* this.parseChild(0, _.assign({}, input, {result: {}}), options)
  }

  *parseChild(childIndex, input, options) {
    if (childIndex >= this.props.children.length) {
      yield input
      return
    }

    const child = this.props.children[childIndex]

    for (let output of parse({phrase: this.childPhrases[childIndex], input, options})) {
      let accumulatedResult = this.props.value ||
        getAccumulatedResult(input.result, child, output.result)
      const nextOutput = _.assign({}, output, {result: accumulatedResult})

      yield* this.parseChild(childIndex + 1, nextOutput, options)
    }

    if (child.props && child.props.optional) {
      yield* this.parseChild(childIndex + 1, input, options)
    }
  }

}


function getAccumulatedResult(inputResult, child, childResult) {
  if (!_.isUndefined(childResult)) {
    const childId = child.props && child.props.id
    const childMerge = child.props && child.props.merge
    if (childId) {
       return _.merge({}, inputResult, {[childId]: childResult})
    } else if (childMerge) {
      return _.merge({}, inputResult, childResult)
    }
  }
  return inputResult
}<commit_msg>Perf improvement for sequence<commit_after>/** @jsx createElement */
import _ from 'lodash'
import {createElement, Phrase} from 'lacona-phrase'
import parse from '../parse'
import {reconcile} from '../reconcile'

function addSeparator (child, separator) {
  if (child.props && child.props.optional) {
    const newChild = _.merge({}, child, {props: {optional: false}})

    return <Sequence optional={true} merge={true}>{newChild}{separator}</Sequence>
  } else {
    return <Sequence merge={true}>{child}{separator}</Sequence>
  }
}

export default class Sequence extends Phrase {
  describe() {
    let content, separator
    if (this.props.children[0] && this.props.children[0].Constructor === 'content') {
      content = this.props.children[0].children
      if (this.props.children[1] && this.props.children[1].Constructor === 'separator') {
        separator = this.props.children[1].children[0]
      }
    } else {
      return //no content, we're good to go!
    }

    if (separator) {
      return (
        <sequence {...this.props}>
          {_.chain(content.slice(0, -1))
            .map(_.partial(addSeparator, _, separator))
            .concat(_.last(content))
            .value()
          }
        </sequence>
      )
    } else {
      return <sequence {...this.props}>{content}</sequence>
    }
  }

  *_handleParse(input, options) {
    this.childPhrases = reconcile({descriptor: this.props.children, phrase: this.childPhrases, options})

    yield* this.parseChild(0, _.assign({}, input, {result: {}}), options)
  }

  *parseChild(childIndex, input, options) {
    if (childIndex >= this.props.children.length) {
      yield input
      return
    }

    const child = this.props.children[childIndex]

    for (let output of parse({phrase: this.childPhrases[childIndex], input, options})) {
      let accumulatedResult = this.props.value ||
        getAccumulatedResult(input.result, child, output.result)
      const nextOutput = _.assign({}, output, {result: accumulatedResult})

      yield* this.parseChild(childIndex + 1, nextOutput, options)
    }

    if (child.props && child.props.optional) {
      yield* this.parseChild(childIndex + 1, input, options)
    }
  }

}


function getAccumulatedResult(inputResult, child, childResult) {
  if (!_.isUndefined(childResult) && child.props) {
    const childId = child.props.id
    const childMerge = child.props.merge
    if (childId) {
       return _.assign({}, inputResult, {[childId]: childResult})
    } else if (childMerge) {
      return _.merge({}, inputResult, childResult)
    }
  }
  return inputResult
}
{'fraction_non_alphanumeric': 0.14701348747591522, 'fraction_numerical': 0.004624277456647399, 'mean_word_length': 3.0777690494893952, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6242627', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1538, 'n_words': 354}