<commit_before>#!/usr/bin/env bash

set -euo pipefail

psql -v ON_ERROR_STOP=1 -v ${variables} -f "$query" "$PGDATABASE" > raw.json
json2fasta.py raw.json rnacentral.fasta
seqkit shuffle --two-pass rnacentral.fasta > shuffled.fasta
seqkit split --two-pass --by-size ${chunk_size} --out-dir 'parts/' shuffled.fasta<commit_msg>Remove extra option<commit_after>#!/usr/bin/env bash

set -euo pipefail

psql -v ON_ERROR_STOP=1 ${variables} -f "$query" "$PGDATABASE" > raw.json
json2fasta.py raw.json rnacentral.fasta
seqkit shuffle --two-pass rnacentral.fasta > shuffled.fasta
seqkit split --two-pass --by-size ${chunk_size} --out-dir 'parts/' shuffled.fasta
{'fraction_non_alphanumeric': 0.14854517611026033, 'fraction_numerical': 0.006125574272588055, 'mean_word_length': 4.404958677685951, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6411224', 'n_tokens_mistral': 255, 'n_tokens_neox': 248, 'n_words': 60}