<commit_before>STEP_1_DIR=step_1
STEP_2_DIR=step_2
STEP_3_DIR=step_3
STEP_4_DIR=step_4
STEP_5_DIR=step_5
STEP_6_DIR=step_6
STEP_7_DIR=step_7
STEP_7_SCRATCH_DIR=step_7_scratch
STEP_8_DIR=step_8
STEP_9_DIR=step_9
SUBPROB_RAW_EXPORT_DIR=$(STEP_7_SCRATCH_DIR)/export-sub-temp
TREE_RANKING_FILE=$(STEP_1_DIR)/tree-ranking.txt
PHYLO_INPUT_FILENAMES=$(shell cat $(TREE_RANKING_FILE))
PHYLO_TREE_NAMES_FILENAMES=$(shell sed -e 's/.tre$$/-tree-names.txt/' $(TREE_RANKING_FILE))
PHYLO_MD5_FILENAMES=$(shell sed -e '/.tre$$/.md5/' $(TREE_RANKING_FILE))

STEP_4_PHYLO_OUTPUT_PATHS=$(addprefix $(STEP_4_DIR)/, $(PHYLO_INPUT_FILENAMES))
STEP_5_PHYLO_INPUT_PATHS=$(STEP_4_PHYLO_OUTPUT_PATHS)
STEP_5_PHYLO_OUTPUT_PATHS=$(addprefix $(STEP_5_DIR)/, $(PHYLO_INPUT_FILENAMES))
STEP_7_PHYLO_INPUT_PATHS=$(STEP_5_PHYLO_OUTPUT_PATHS)
STEP_7_PHYLO_OUTPUT_TREE_PATHS=$(addprefix $(STEP_7_DIR)/, $(PHYLO_INPUT_FILENAMES))
STEP_7_PHYLO_OUTPUT_TREE_NAME_PATHS=$(addprefix $(STEP_7_DIR)/, $(PHYLO_TREE_NAMES_FILENAMES))
STEP_7_PHYLO_OUTPUT_MD5_PATHS=$(addprefix $(STEP_7_DIR)/, $(PHYLO_MD5_FILENAMES))

# Eventually $(STEP_4_PHYLO_OUTPUT_PATHS) should be added to the artifact list...
ARTIFACTS=$(STEP_5_DIR)/step-5-phylo-args.txt \
	$(STEP_5_DIR)/taxonomy.tre \
	$(STEP_6_DIR)/taxonomy.tre \
	$(STEP_7_DIR)/step-7-phylo-args.txt \
	$(STEP_7_SCRATCH_DIR)/dumped-subproblem-ids.txt \
	$(STEP_7_SCRATCH_DIR)/checksummed-subproblem-ids.txt \
	$(STEP_7_DIR)/subproblem-ids.txt \
	$(STEP_5_PHYLO_OUTPUT_PATHS) \
	$(STEP_7_PHYLO_OUTPUT_TREE_PATHS) \
	$(STEP_7_PHYLO_OUTPUT_TREE_NAME_PATHS) \
	$(STEP_7_PHYLO_OUTPUT_MD5_PATHS)<commit_msg>accidentally had dropped s from sed command<commit_after>STEP_1_DIR=step_1
STEP_2_DIR=step_2
STEP_3_DIR=step_3
STEP_4_DIR=step_4
STEP_5_DIR=step_5
STEP_6_DIR=step_6
STEP_7_DIR=step_7
STEP_7_SCRATCH_DIR=step_7_scratch
STEP_8_DIR=step_8
STEP_9_DIR=step_9
SUBPROB_RAW_EXPORT_DIR=$(STEP_7_SCRATCH_DIR)/export-sub-temp
TREE_RANKING_FILE=$(STEP_1_DIR)/tree-ranking.txt
PHYLO_INPUT_FILENAMES=$(shell cat $(TREE_RANKING_FILE))
PHYLO_TREE_NAMES_FILENAMES=$(shell sed -e 's/.tre$$/-tree-names.txt/' $(TREE_RANKING_FILE))
PHYLO_MD5_FILENAMES=$(shell sed -e 's/.tre$$/.md5/' $(TREE_RANKING_FILE))

STEP_4_PHYLO_OUTPUT_PATHS=$(addprefix $(STEP_4_DIR)/, $(PHYLO_INPUT_FILENAMES))
STEP_5_PHYLO_INPUT_PATHS=$(STEP_4_PHYLO_OUTPUT_PATHS)
STEP_5_PHYLO_OUTPUT_PATHS=$(addprefix $(STEP_5_DIR)/, $(PHYLO_INPUT_FILENAMES))
STEP_7_PHYLO_INPUT_PATHS=$(STEP_5_PHYLO_OUTPUT_PATHS)
STEP_7_PHYLO_OUTPUT_TREE_PATHS=$(addprefix $(STEP_7_DIR)/, $(PHYLO_INPUT_FILENAMES))
STEP_7_PHYLO_OUTPUT_TREE_NAME_PATHS=$(addprefix $(STEP_7_DIR)/, $(PHYLO_TREE_NAMES_FILENAMES))
STEP_7_PHYLO_OUTPUT_MD5_PATHS=$(addprefix $(STEP_7_DIR)/, $(PHYLO_MD5_FILENAMES))

# Eventually $(STEP_4_PHYLO_OUTPUT_PATHS) should be added to the artifact list...
ARTIFACTS=$(STEP_5_DIR)/step-5-phylo-args.txt \
	$(STEP_5_DIR)/taxonomy.tre \
	$(STEP_6_DIR)/taxonomy.tre \
	$(STEP_7_DIR)/step-7-phylo-args.txt \
	$(STEP_7_SCRATCH_DIR)/dumped-subproblem-ids.txt \
	$(STEP_7_SCRATCH_DIR)/checksummed-subproblem-ids.txt \
	$(STEP_7_DIR)/subproblem-ids.txt \
	$(STEP_5_PHYLO_OUTPUT_PATHS) \
	$(STEP_7_PHYLO_OUTPUT_TREE_PATHS) \
	$(STEP_7_PHYLO_OUTPUT_TREE_NAME_PATHS) \
	$(STEP_7_PHYLO_OUTPUT_MD5_PATHS)
{'fraction_non_alphanumeric': 0.13230293663060277, 'fraction_numerical': 0.03400309119010819, 'mean_word_length': 9.750830564784053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7347624', 'n_tokens_mistral': 1542, 'n_tokens_neox': 1525, 'n_words': 129}