<commit_before>gzip -cd *_R3.fastq.* | head -1000000 | awk 'NR == 2 || NR % 4 == 2' | grep -v N | sort | uniq -c | sort -nr | head -n 24 > sample_barcodes.log<commit_msg>Improve paths to work from project root<commit_after>mkdir -p logs
gzip -cd data-raw/fastq/*_R3.fastq.* | \
head -1000000 | \
awk 'NR == 2 || NR % 4 == 2' | \
grep -v N | \
sort | \
uniq -c | \
sort -nr | \
head -n 24 > logs/sample_barcodes.log
{'fraction_non_alphanumeric': 0.178743961352657, 'fraction_numerical': 0.06280193236714976, 'mean_word_length': 2.608695652173913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2722954', 'n_tokens_mistral': 193, 'n_tokens_neox': 166, 'n_words': 54}