<commit_before># Illustrate the simplest possible thing where a CSV output files is
# generated by some script. The command is rendered with jinja based
# on the rest of the keys in that YAML object.
---
creates: data/first.csv
depends: ./src/script.sh
command: {{depends}} > {{creates}}


# This is another pretty common use case that illustrates a dependency
# on another step (in this case, creating data/first.csv). It also
# illustrates how to specify creating a mysql table and how to
# dereference different parts of the dependency chain in the command
# using Jinja templating.
---
creates: mysql:mydb/mytable
depends: 
  - data/first.csv
  - ./src/fill_mytable.py
command: {{depends[1]}} < {{depends[0]}}


# create a directory of files. expand {{depends}} in order when a list
---
creates: data/first/
depends: 
  - src/new_file_for_every_row.py
  - data/first.csv
command: python {{depends}} -o {{creates}}


# show how things can depend on multiple steps
---
creates: data/result.png
depends:
  - src/render_result.py
  - data/first.csv
  - data/first/
command: python {{depends}} > {{creates}}


# Illustrate how to run a per-file analysis that is stupidly parallel
# using regex matching to explicitly demonstrate how filenames are
# manipulated. This should make it possible to run our workflow on
# each file separately in addition to running this analysis on every
# file in the directory.
# 
# [unix]$ workflow data/per_file
# [unix]$ workflow data/per_file --filename-root=a_specific_filename
# [unix]$ workflow data/per_file --time-limit=1s
---
alias: data/per_file
creates: data/per_file/{{filename_root}}.dat
depends:
  - src/process_each_file.py
  - data/first/(?P<filename_root>\w+).dat
  - data/first/
command: python {{depends[0]}} {{depends[1]}} > {{creates}}


# include a suite of analyses from a different directory see
# http://stackoverflow.com/a/9577670/564709
!include workflow/images.yaml


# show how to run multiple commands. Treat a list of commands as
# creating a shell script after rendering the Jinja template
---
creates: data/probability_density.dat
depends: data/first.csv
command:
  - sed 's/[,\n\r]/ /g' < {{depends}} > kk
  - sort -nr2,2k kk > gg<commit_msg>Update workflow.yaml<commit_after># Illustrate the simplest possible thing where a CSV output file is
# generated by some script. The command is rendered with jinja based templating
# on the rest of the keys in that YAML object.
---
creates: data/first.csv
depends: ./src/script.sh
command: {{depends}} > {{creates}}


# This is another pretty common use case that illustrates a dependency
# on another step (in this case, creating data/first.csv). It also
# illustrates how to specify creating a mysql table and how to
# dereference different parts of the dependency chain in the command
# using Jinja templating.
---
creates: mysql:mydb/mytable
depends: 
  - data/first.csv
  - ./src/fill_mytable.py
command: {{depends[1]}} {{creates}} < {{depends[0]}}


# create a directory of files. expand `depends` using default jinja join filter.
---
creates: data/first/
depends: 
  - src/new_file_for_every_row.py
  - data/first.csv
command: python {{depends|join(' ')}} -o {{creates}}


# show how things can depend on multiple steps
---
creates: data/result.png
depends:
  - src/render_result.py
  - data/first.csv
  - data/first/
command: python {{depends|join(' ')}} > {{creates}}


# Illustrate how to run a per-file analysis that is stupidly parallel
# using regex matching to explicitly demonstrate how filenames are
# manipulated. This should make it possible to run our workflow on
# each file separately in addition to running this analysis on every
# file in the directory.
# The idea here/ the use case in mind here is to allow analysis to be
# easily run on one file or all the files in a direcotry.
# 
# COMMAND LINE INTERFACE EXAMPLES:
# [unix]$ workflow data/per_file
# [unix]$ workflow data/per_file --filename-root=a_specific_filename
# [unix]$ workflow data/per_file --time-limit=1s
---
alias: data/per_file
creates: data/per_file/{{filename_root}}.dat
depends:
  - src/process_each_file.py
  - data/first/(?P<filename_root>\w+).dat
  - data/first/
command: python {{depends[0]}} {{depends[1]}} > {{creates}}


# include a suite of analyses from a different directory see
# http://stackoverflow.com/a/9577670/564709
!include workflow/images.yaml


# show how to run multiple commands. Treat a list of commands as
# creating a shell script after rendering the Jinja template
---
creates: data/probability_density.dat
depends: data/first.csv
command:
  - sed 's/[,\n\r]/ /g' < {{depends}} > kk
  - sort -nr2,2k kk > gg
{'fraction_non_alphanumeric': 0.11350884764782046, 'fraction_numerical': 0.008631851532153647, 'mean_word_length': 3.973175965665236, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5773903', 'n_tokens_mistral': 1500, 'n_tokens_neox': 1438, 'n_words': 535}