<commit_before>import os
import json
import logging
from itertools import chain

from data import path_to_file, Data
from transforms import TRANSFORMS
from graph import render_pipeline, pipeline_md_for_name


def execute_pipeline(repo_path, pipeline_name, pipeline_args=[], cache_stages=True, force_render_metrics=False):
    with open(path_to_file(repo_path, 'pipeline.json'), 'r+') as config_file:
        config = json.loads(config_file.read(), encoding='utf-8')
        config['repo_path'] = repo_path

        init_env(repo_path)

        transforms = []
        named_stages = config['pipelines'].get(pipeline_name, None)

        if not named_stages:
            if len(config['pipelines']) == 1:
                named_stages = config['pipelines'].keys()[0]['stages']
            else:
                raise ValueError(
                    "Pipeline.json does not contain a stage named '{0}'"
                    .format(pipeline_name)
                )
        else:
            pipeline_is_ephemeral = named_stages.get('ephemeral', False)
            named_stages = named_stages['stages']

        for stage in named_stages:
            module_name, class_name = TRANSFORMS[config['transforms'][stage['transform']]['transform_type']].rsplit('.',1)
            module = __import__(module_name, fromlist=[class_name])
            transform_cls = getattr(module, class_name)
            transform = transform_cls(stage, config, pipeline_is_ephemeral)
            transforms.append(transform)<commit_msg>access correct dictionary key<commit_after>import os
import json
import logging
from itertools import chain

from data import path_to_file, Data
from transforms import TRANSFORMS
from graph import render_pipeline, pipeline_md_for_name


def execute_pipeline(repo_path, pipeline_name, pipeline_args=[], cache_stages=True, force_render_metrics=False):
    with open(path_to_file(repo_path, 'pipeline.json'), 'r+') as config_file:
        config = json.loads(config_file.read(), encoding='utf-8')
        config['repo_path'] = repo_path

        init_env(repo_path)

        transforms = []
        named_stages = config['pipelines'].get(pipeline_name, None)

        if not named_stages:
            if len(config['pipelines']) == 1:
                named_stages = config['pipelines'][config['pipelines'].keys()[0]]['stages']
            else:
                raise ValueError(
                    "Pipeline.json does not contain a stage named '{0}'"
                    .format(pipeline_name)
                )
        else:
            pipeline_is_ephemeral = named_stages.get('ephemeral', False)
            named_stages = named_stages['stages']

        for stage in named_stages:
            module_name, class_name = TRANSFORMS[config['transforms'][stage['transform']]['transform_type']].rsplit('.',1)
            module = __import__(module_name, fromlist=[class_name])
            transform_cls = getattr(module, class_name)
            transform = transform_cls(stage, config, pipeline_is_ephemeral)
            transforms.append(transform)
{'fraction_non_alphanumeric': 0.10409165302782324, 'fraction_numerical': 0.0032733224222585926, 'mean_word_length': 2.9381443298969074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3993946', 'n_tokens_mistral': 885, 'n_tokens_neox': 863, 'n_words': 182}