<commit_before>machine:
  environment:
    PATH: ~/.local/bin:~/spark/bin:$PATH
  services:
  - docker

checkout:
  post:
  - git submodule update --recursive --init

dependencies:
  cache_directories:
  - "~/.stack"

  override:
  - mkdir -p ~/.local/bin
  - curl -L --retry 3 https://www.stackage.org/stack/linux-x86_64 | tar xz --wildcards --strip-components=1 -C ~/.local/bin '*/stack'
  - docker build -t sparkle .
  - stack --no-terminal --docker --docker-image sparkle build --only-snapshot --prefetch

test:
  override:
  - stack --no-terminal --docker --docker-image sparkle build --pedantic
  # Test that packaging at least one example works fine.
  - stack --no-terminal --docker --docker-image sparkle exec sparkle package sparkle-example-hello
<commit_msg>CI: check that the packaged example runs fine<commit_after>machine:
  environment:
    PATH: ~/.local/bin:~/spark/bin:$PATH
  services:
  - docker

checkout:
  post:
  - git submodule update --recursive --init

dependencies:
  cache_directories:
  - "~/.stack"

  override:
  - mkdir -p ~/.local/bin
  - curl -L --retry 3 https://www.stackage.org/stack/linux-x86_64 | tar xz --wildcards --strip-components=1 -C ~/.local/bin '*/stack'
  - docker build -t sparkle .
  - stack --no-terminal --docker --docker-image sparkle build --only-snapshot --prefetch

test:
  override:
  - stack --no-terminal --docker --docker-image sparkle build --pedantic
  # Test that packaging at least one example works fine.
  - stack --no-terminal --docker --docker-image sparkle exec sparkle package sparkle-example-hello
  - spark-submit --master 'local[1]' sparkle-example-hello.jar

{'fraction_non_alphanumeric': 0.14329454990814453, 'fraction_numerical': 0.007960808328230252, 'mean_word_length': 2.817757009345794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2161223', 'n_tokens_mistral': 568, 'n_tokens_neox': 523, 'n_words': 153}