<commit_before>'use strict';

var utils = require('./utils');

function parseTask(str) {
  var arr = str.split(':');
  var key = arr[0];
  var val = arr[1];
  var res = {};

  if (arr.length === 1) {
    utils.union(res, 'base', key);
  } else if (key === 'base') {
    utils.union(res, 'base', val);
  } else {
    key = key.split('.').join('\\.');
    utils.union(res, key, val.split(','));
  }
  return res;
}

function toTasks(tasks) {
  if (typeof tasks === 'string') {
    if (!/[\W.]/.test(tasks)) {
      return { base: [tasks] };
    }
  }

  if (Array.isArray(tasks)) {
    return tasks.reduce(function(acc, str) {
      return acc.concat(toTasks(str));
    }, []);
  }

  var arr = tasks.split('|');
  var len = arr.length, i = -1;
  var res = [];

  while (++i < len) {
    var obj = parseTask(arr[i]);
    res.push(obj);<commit_msg>ensure tasks are split<commit_after>'use strict';

var utils = require('./utils');

function parseTask(str) {
  var arr = str.split(':');
  var key = arr[0];
  var val = arr[1];
  var res = {};

  if (arr.length === 1) {
    utils.union(res, 'base', key);
  } else if (key === 'base') {
    utils.union(res, 'base', val.split(','));
  } else {
    key = key.split('.').join('\\.');
    utils.union(res, key, val.split(','));
  }

  return res;
}

function toTasks(tasks) {
  if (typeof tasks === 'string') {
    if (!/[\W.]/.test(tasks)) {
      return { base: tasks.split(',') };
    }
  }

  if (Array.isArray(tasks)) {
    return tasks.reduce(function(acc, str) {
      return acc.concat(toTasks(str));
    }, []);
  }

  var arr = tasks.split('|');
  var len = arr.length, i = -1;
  var res = [];

  while (++i < len) {
    var obj = parseTask(arr[i]);
    res.push(obj);
{'fraction_non_alphanumeric': 0.2302325581395349, 'fraction_numerical': 0.004651162790697674, 'mean_word_length': 2.685224839400428, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6562938', 'n_tokens_mistral': 650, 'n_tokens_neox': 644, 'n_words': 150}