<commit_before>import io
import os
import json
import re

from ..dialect import Dialect

def compile(feature, path):
    pickles          = []
    dialect          = feature['language']
    feature_tags     = feature['tags']
    background_steps = _get_background_steps(feature, path)
    for scenario_definition in feature['scenarioDefinitions']:
        args = feature_tags, background_steps, scenario_definition, dialect, path, pickles
        if scenario_definition['type'] is 'Scenario':
            _compile_scenario(*args)
        else:
            _compile_scenario_outline(*args)
    return pickles

def _compile_scenario(feature_tags, background_steps, scenario, dialect, path, pickles):
    steps = list(background_steps)
    tags  = list(feature_tags) + list(scenario['tags'])
    
    for step in scenario['steps']:
        steps.append(_pickle_step(step, path))

    pickle = {
        'path':     path,
        'tags':     _pickle_tags(tags, path),
        'name':     '{0[keyword]}: {0[name]}'.format(scenario),
        'locations':[_pickle_location(scenario['location'], path)],
        'steps':    steps
    }
    pickles.append(pickle)

def _compile_scenario_outline(feature_tags, background_steps, scenario_outline, dialect, path, pickles):
    keyword = Dialect.for_name(dialect).scenario_keywords[0]
    
    for examples in scenario_outline['examples']:
        variable_cells = examples['tableHeader']['cells']
        
        for values in examples['tableBody']:
            value_cells = values['cells']
            steps       = list(background_steps)
            tags        = list(feature_tags) + list(scenario_outline['tags']) + list(examples['tags'])
            
            for scenario_outline_step in scenario_outline['steps']:
                step_text = _interpolate(scenario_outline_step['text'], variable_cells, value_cells)
                arguments = _create_pickle_arguments(scenario_outline_step.get('argument'), variable_cells, value_cells, path)
                _pickle_step = {
                    'text': step_text,
                    'arguments': arguments,
                    'locations': [
                        _pickle_location(values['location'], path),
                        _pickle_step_location(scenario_outline_step, path)
                    ]
                }
                steps.append(_pickle_step)

            pickle = {
                'path':  path,
                'name':  '{0}: {1}'.format(keyword, _interpolate(scenario_outline['name'], variable_cells, value_cells)),
                'steps': steps,
                'tags':  _pickle_tags(tags, path),
                'locations': [
                    _pickle_location(values['location'], path),
                    _pickle_location(scenario_outline['location'], path)
                ]
            }
            pickles.append(pickle)

def _create_pickle_arguments(argument, variables, values, path):
    result = []
    
    if not argument:
        return result
    
    if argument['type'] is 'DataTable':
        table = { 'rows': [] }
        for row in argument['rows']:
            cells = [
                {   
                    'location': _pickle_location(cell['location'], path),
                    'value':    _interpolate(cell['value'], variables, values) 
                } for cell in row['cells']
            ]
            table['rows'].append({'cells': cells})
        result.append(table)
    
    elif argument['type'] is 'DocString':
        docstring = {
            'location': _pickle_location(argument['location'], path),
            'content':  _interpolate(argument['content'], variables, values)
        }
        result.append(docstring)
    
    else:
        raise Exception('Internal error')
    
    return result

def _interpolate(name, variable_cells, value_cells):
    for n, variable_cell in enumerate(variable_cells):
        value_cell  = value_cells[n]
        name        = re.sub( 
            '<{0[value]}>'.format(variable_cell),
            value_cell['value'],
            name
            )
    return name

def _get_background_steps(feature, path):
    if 'background' in feature:
        return [_pickle_step(step, path) for step in feature['background']['steps']]
    return []

def _pickle_step(step, path):
    return {
        'text':      step['text'],
        'arguments': _create_pickle_arguments(step.get('argument'), [], [], path),
        'locations': [_pickle_step_location(step, path)]
    }

def _pickle_step_location(step, path):
    return {
        'path':   path,
        'line':   step['location']['line'],
        'column': step['location']['column'] + len( step.get('keyword', 0) )
    }

def _pickle_location(location, path):
    return {
        'path':   path,
        'line':   location['line'],
        'column': location['column']
    }

def _pickle_tags(tags, path):
    return [_pickle_tag(tag, path) for tag in tags]

def _pickle_tag(tag, path):
    return {
        'name':     tag['name'],
        'location': _pickle_location(tag['location'], path)
    }<commit_msg>(Python) PEP8 for gherkin3.pickles.compiler<commit_after>import io
import os
import json
import re

from ..dialect import Dialect


def compile(feature, path):
    pickles = []
    dialect = feature['language']
    feature_tags = feature['tags']
    background_steps = _get_background_steps(feature, path)
    for scenario_definition in feature['scenarioDefinitions']:
        args = (feature_tags, background_steps, scenario_definition, dialect,
                path, pickles)
        if scenario_definition['type'] is 'Scenario':
            _compile_scenario(*args)
        else:
            _compile_scenario_outline(*args)
    return pickles


def _compile_scenario(feature_tags, background_steps, scenario, dialect, path,
                      pickles):
    steps = list(background_steps)
    tags = list(feature_tags) + list(scenario['tags'])

    for step in scenario['steps']:
        steps.append(_pickle_step(step, path))

    pickle = {
        'path': path,
        'tags': _pickle_tags(tags, path),
        'name': '{0[keyword]}: {0[name]}'.format(scenario),
        'locations': [_pickle_location(scenario['location'], path)],
        'steps': steps
    }
    pickles.append(pickle)


def _compile_scenario_outline(feature_tags, background_steps,
                              scenario_outline, dialect, path, pickles):
    keyword = Dialect.for_name(dialect).scenario_keywords[0]

    for examples in scenario_outline['examples']:
        variable_cells = examples['tableHeader']['cells']

        for values in examples['tableBody']:
            value_cells = values['cells']
            steps = list(background_steps)
            tags = list(feature_tags) \
                + list(scenario_outline['tags']) \
                + list(examples['tags'])

            for scenario_outline_step in scenario_outline['steps']:
                step_text = _interpolate(
                    scenario_outline_step['text'],
                    variable_cells,
                    value_cells)
                arguments = _create_pickle_arguments(
                    scenario_outline_step.get('argument'),
                    variable_cells,
                    value_cells,
                    path)
                _pickle_step = {
                    'text': step_text,
                    'arguments': arguments,
                    'locations': [
                        _pickle_location(values['location'], path),
                        _pickle_step_location(scenario_outline_step, path)
                    ]
                }
                steps.append(_pickle_step)

            pickle = {
                'path': path,
                'name': '{0}: {1}'.format(
                    keyword,
                    _interpolate(
                        scenario_outline['name'],
                        variable_cells,
                        value_cells)
                    ),
                'steps': steps,
                'tags': _pickle_tags(tags, path),
                'locations': [
                    _pickle_location(values['location'], path),
                    _pickle_location(scenario_outline['location'], path)
                ]
            }
            pickles.append(pickle)


def _create_pickle_arguments(argument, variables, values, path):
    result = []

    if not argument:
        return result

    if argument['type'] is 'DataTable':
        table = {'rows': []}
        for row in argument['rows']:
            cells = [
                {
                    'location': _pickle_location(cell['location'], path),
                    'value': _interpolate(cell['value'], variables, values)
                } for cell in row['cells']
            ]
            table['rows'].append({'cells': cells})
        result.append(table)

    elif argument['type'] is 'DocString':
        docstring = {
            'location': _pickle_location(argument['location'], path),
            'content': _interpolate(argument['content'], variables, values)
        }
        result.append(docstring)

    else:
        raise Exception('Internal error')

    return result


def _interpolate(name, variable_cells, value_cells):
    for n, variable_cell in enumerate(variable_cells):
        value_cell = value_cells[n]
        name = re.sub(
            '<{0[value]}>'.format(variable_cell),
            value_cell['value'],
            name
            )
    return name


def _get_background_steps(feature, path):
    if 'background' in feature:
        return [_pickle_step(step, path)
                for step in feature['background']['steps']]
    return []


def _pickle_step(step, path):
    return {
        'text': step['text'],
        'arguments': _create_pickle_arguments(
            step.get('argument'),
            [],
            [],
            path),
        'locations': [_pickle_step_location(step, path)]
    }


def _pickle_step_location(step, path):
    return {
        'path': path,
        'line': step['location']['line'],
        'column': step['location']['column'] + len(step.get('keyword', 0))
    }


def _pickle_location(location, path):
    return {
        'path': path,
        'line': location['line'],
        'column': location['column']
    }


def _pickle_tags(tags, path):
    return [_pickle_tag(tag, path) for tag in tags]


def _pickle_tag(tag, path):
    return {
        'name': tag['name'],
        'location': _pickle_location(tag['location'], path)
    }
{'fraction_non_alphanumeric': 0.11931493815413892, 'fraction_numerical': 0.001522359657469077, 'mean_word_length': 2.0644314868804665, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6932064', 'n_tokens_mistral': 3000, 'n_tokens_neox': 2858, 'n_words': 584}