<commit_before>'use strict';

import invariant from 'invariant';

import Registry from '../common/MultivalueRegistry';
import { isElementRef } from '../data/element';

import { Iterable } from 'immutable';

import R from 'ramda';

import { postWalk } from 'zippa'
import elementZipper from '../zip/elementZipper'

const transformerRegistry = new Registry();


export function register(kind, transformer) {
  invariant(
    isElementRef(kind),
    "You must provide a valid element reference to register");
  invariant(
    transformer != null && typeof transformer === 'function',
    "You must provide a transformer function"
  );

  transformerRegistry.register(kind, transformer);
}

export function get(kind) {
  return transformerRegistry.get(kind)
}


function transform(element) {

  // this is a safety check for the case when some of the children elements also matches a field with a scalar value
  if (!Iterable.isIndexed(element) && !Iterable.isAssociative(element)) {
    return false
  }

  // get all transformers registered for this kind
  const transformers = get(element.get('kind'))

  // if no transformers are defined for the element, just return the same element
  if (!transformers) {
    return element
  }

  const transformFn = transformers.length === 1 ? transformers : R.pipe(...transformers)
  return transformFn(element)
}

export function apply(element, childrenElements = 'content') {
  let elementToProcess = element
  // TODO: implement proper handling for the case when the root is an array with multiple elements
  if (Iterable.isIndexed(element)) {
    elementToProcess = element.get(0)
  }
  const zipper = elementZipper(elementToProcess, childrenElements)
  return postWalk(transform, zipper)
}

export function reset() {
  transformerRegistry.reset();
}<commit_msg>Fix bug - return the unmodified element when it is a child with scalar value, instead of false, which causes the element to be removed from the application state<commit_after>'use strict';

import invariant from 'invariant';

import Registry from '../common/MultivalueRegistry';
import { isElementRef } from '../data/element';

import { Iterable } from 'immutable';

import R from 'ramda';

import { postWalk } from 'zippa'
import elementZipper from '../zip/elementZipper'

const transformerRegistry = new Registry();


export function register(kind, transformer) {
  invariant(
    isElementRef(kind),
    "You must provide a valid element reference to register");
  invariant(
    transformer != null && typeof transformer === 'function',
    "You must provide a transformer function"
  );

  transformerRegistry.register(kind, transformer);
}

export function get(kind) {
  return transformerRegistry.get(kind)
}


function transform(element) {

  // this is a safety check for the case when some of the children elements also matches a field with a scalar value
  if (!Iterable.isIndexed(element) && !Iterable.isAssociative(element)) {
    return element
  }

  // get all transformers registered for this kind
  const transformers = get(element.get('kind'))

  // if no transformers are defined for the element, just return the same element
  if (!transformers) {
    return element
  }

  const transformFn = transformers.length === 1 ? transformers : R.pipe(...transformers)
  return transformFn(element)
}

export function apply(element, childrenElements = 'content') {
  let elementToProcess = element
  // TODO: implement proper handling for the case when the root is an array with multiple elements
  if (Iterable.isIndexed(element)) {
    elementToProcess = element.get(0)
  }
  const zipper = elementZipper(elementToProcess, childrenElements)
  return postWalk(transform, zipper)
}

export function reset() {
  transformerRegistry.reset();
}
{'fraction_non_alphanumeric': 0.09552953698775944, 'fraction_numerical': 0.0010643959552953698, 'mean_word_length': 4.385386819484241, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6264298', 'n_tokens_mistral': 1020, 'n_tokens_neox': 988, 'n_words': 369}