<commit_before>"use strict";

//pipe a stream to another, streams might be promises

function pipe(src, dest) {

  if(src.then)
    return src.then(function(tmp) {
      return pipe(tmp, dest);
    });

  if(dest.then)
    return dest.then(function(tmp) {
      return pipe(src, tmp);
    });

  return new Promise(function(resolve, reject) {
    src.pipe(dest);
    src.on('error', reject);
    dest.on('close', resolve);
  });
<commit_msg>fix stream/pipe<commit_after>"use strict";

//pipe a stream to another, streams might be promises

function pipe(src, dest) {

  if(src.then)
    return src.then(function(tmp) {
      return pipe(tmp, dest);
    });

  if(dest.then)
    return dest.then(function(tmp) {
      return pipe(src, tmp);
    });

  return new Promise(function(resolve, reject) {
    src.pipe(dest);
    src.on('error', reject);
    dest.on('finish', resolve);
    dest.on('close', resolve);
  });

{'fraction_non_alphanumeric': 0.15938864628820962, 'fraction_numerical': 0.0, 'mean_word_length': 3.206422018348624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6150093', 'n_tokens_mistral': 306, 'n_tokens_neox': 299, 'n_words': 73}