<commit_before>name: Demo pipeline - Total size of two files
steps:
  -
    name: Get size of file 1
    image: dleehr/filesize
    infiles:
      CONT_INPUT_FILE: !var FILE1
    outfiles:
      CONT_OUTPUT_FILE: /data/step1/size
  -
    name: Get size of file 2
    image: dleehr/filesize
    infiles:
      CONT_INPUT_FILE: !var FILE2
    outfiles:
      CONT_OUTPUT_FILE: /data/step2/size
  -
    name: Add sizes of files
    image: dleehr/add
    infiles:
      CONT_INPUT_FILE1: /data/step1/size
      CONT_INPUT_FILE2: /data/step2/size
    outfiles:
      CONT_OUTPUT_FILE: !var RESULT
<commit_msg>Use tmp paths for intermediate files in sample<commit_after>name: Demo pipeline - Total size of two files
steps:
  -
    name: Get size of file 1
    image: dleehr/filesize
    infiles:
      CONT_INPUT_FILE: !var FILE1
    outfiles:
      CONT_OUTPUT_FILE: /tmp/step1-size
  -
    name: Get size of file 2
    image: dleehr/filesize
    infiles:
      CONT_INPUT_FILE: !var FILE2
    outfiles:
      CONT_OUTPUT_FILE: /tmp/step2-size
  -
    name: Add sizes of files
    image: dleehr/add
    infiles:
      CONT_INPUT_FILE1: /tmp/step1-size
      CONT_INPUT_FILE2: /tmp/step2-size
    outfiles:
      CONT_OUTPUT_FILE: !var RESULT

{'fraction_non_alphanumeric': 0.07437348423605497, 'fraction_numerical': 0.016168148746968473, 'mean_word_length': 2.4775280898876404, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6368471', 'n_tokens_mistral': 456, 'n_tokens_neox': 437, 'n_words': 117}