<commit_before>import ast

import astcheck
import astsearch

from ..code import Parameter

__all__ = ['extract_definitions', 'build_definitions']

def check_fillable_node(node, path):
    if isinstance(node, (ast.Num, ast.Str, ast.List)):
        return
    elif isinstance(node, ast.NameConstant) and (node.value in (True, False)):
        return
    
    raise astcheck.ASTMismatch(path, node, 'number, string or boolean')

definition_pattern = ast.Assign(targets=[ast.Name()], value=check_fillable_node)

def type_and_value(node):
    if isinstance(node, ast.Num):
        # int or float
        return type(node.n), node.n
    elif isinstance(node, ast.Str):
        return str, node.s
    elif isisntance(node, ast.List):
        return list, node.s
    return (bool, node.value)

def extract_definitions(cell):
    cell_ast = ast.parse(cell)
    for assign in astsearch.ASTPatternFinder(definition_pattern).scan_ast(cell_ast):
        yield Parameter(assign.targets[0].id, *type_and_value(assign.value))

def build_definitions(inputs):
    return "\n".join("{0.name} = {0.value!r}".format(i) for i in inputs)<commit_msg>Add lists as valid parameters<commit_after>import ast

import astcheck
import astsearch

from ..code import Parameter

__all__ = ['extract_definitions', 'build_definitions']

def check_list(node):
    def bool_check(node):
        return isinstance(node, ast.NameConstant) and (node.value in (True, False))
    return all([(isinstance(n, (ast.Num, ast.Str))
                 or bool_check(n)) for n in node.elts])

def check_fillable_node(node, path):
    if isinstance(node, (ast.Num, ast.Str)):
        return
    elif (isinstance(node, ast.List)
          and isinstance(node.ctx, ast.Load) and check_list(node)):
        return
    elif isinstance(node, ast.NameConstant) and (node.value in (True, False)):
        return

    raise astcheck.ASTMismatch(path, node, 'number, string, list or boolean')

definition_pattern = ast.Assign(targets=[ast.Name()], value=check_fillable_node)

def type_and_value(node):
    if isinstance(node, ast.Num):
        # int or float
        return type(node.n), node.n
    elif isinstance(node, ast.Str):
        return str, node.s
    elif isinstance(node, ast.List):
        return list, [type_and_value(n)[1] for n in node.elts]
    return (bool, node.value)

def extract_definitions(cell):
    cell_ast = ast.parse(cell)
    for assign in astsearch.ASTPatternFinder(definition_pattern).scan_ast(cell_ast):
        yield Parameter(assign.targets[0].id, *type_and_value(assign.value))

def build_definitions(inputs):
    return "\n".join("{0.name} = {0.value!r}".format(i) for i in inputs)


{'fraction_non_alphanumeric': 0.12344749717726759, 'fraction_numerical': 0.0026345502446368085, 'mean_word_length': 3.7549194991055455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6598381', 'n_tokens_mistral': 853, 'n_tokens_neox': 847, 'n_words': 212}