<commit_before>from types import BeaverException, Variable, Pattern, EmptyPattern, updated_context
from copy import deepcopy as copy


class Statement(object):
    '''A generic statement containing any number of variable parts.'''
    def __init__(self, *parts):
        self.parts = parts
    def __str__(self): 
        return (' '.join([str(part) for part in self.parts[:3]]) + 
                ''.join([', %s' % part for part in self.parts[3:]]))
    def __repr__(self): return str(self)
    def __eq__(self, x): return str(self) == str(x)
    
    def as_triple(self): return TripleStatement(*self.parts)
    
    def replace(self, *varsets):
        '''Checks each part of the statement against defined variables. If any
        matches are found, the statement is updated. If the statement is a function
        call, a new set of statements is returned; otherwise, None is returned.'''
        matched = False
        
        # check for single variables in statement
        for n, part in enumerate(self.parts):
            if isinstance(part, Variable):
                for varset in varsets:
                    if matched: break
                    if part in varset:
                        defs = varset[part]
                        for (pattern, definition) in defs:
                            if matched: break
                            if part == definition: continue
                            
                            if len(pattern.vars) == 0:
                                definition = copy(definition)
                                
                                # a match was found; replace the variable with its definition
                                if isinstance(definition, Statement): definition = definition.parts
                                elif isinstance(definition, list): definition = [p for stmt in definition for p in stmt.parts]
                                else: definition = [definition]
                                
                                self.parts = self.parts[:n] + tuple(definition) + self.parts[n+1:]
                                matched = True
                
        # repeat until no changes were made
        if matched: return self.replace(*varsets)
        
        # check if statement is a function call
        if isinstance(self.parts[0], Variable):
            var = self.parts[0]
            args = self.parts[1:]<commit_msg>Smarter variable lookup handling involving lists.<commit_after>from types import BeaverException, Variable, Pattern, EmptyPattern, updated_context
from copy import deepcopy as copy


class Statement(object):
    '''A generic statement containing any number of variable parts.'''
    def __init__(self, *parts):
        self.parts = list(parts)
    def __str__(self): 
        return (' '.join([str(part) for part in self.parts[:3]]) + 
                ''.join([', %s' % part for part in self.parts[3:]]))
    def __repr__(self): return str(self)
    def __eq__(self, x): return str(self) == str(x)
    
    def as_triple(self): return TripleStatement(*self.parts)
    
    def replace(self, *varsets):
        '''Checks each part of the statement against defined variables. If any
        matches are found, the statement is updated. If the statement is a function
        call, a new set of statements is returned; otherwise, None is returned.'''
        matched = False
        
        # check for single variables in statement
        for n, part in enumerate(self.parts):
            if isinstance(part, Variable):
                for varset in varsets:
                    if matched: break
                    if part in varset:
                        defs = varset[part]
                        for (pattern, definition) in defs:
                            if matched: break
                            if part == definition: continue
                            
                            if len(pattern.vars) == 0:
                                definition = copy(definition)
                                
                                # a match was found; replace the variable with its definition
                                if isinstance(definition, tuple): definition = list(definition)
                                
                                if isinstance(definition, Statement): definition = definition.parts
                                elif isinstance(definition, list): definition = [p for stmt in definition for p in stmt.parts]
                                else: definition = [definition]
                                
                                self.parts = self.parts[:n] + definition + self.parts[n+1:]
                                matched = True
                
        # repeat until no changes were made
        if matched: return self.replace(*varsets)
        
        # check if statement is a function call
        if isinstance(self.parts[0], Variable):
            var = self.parts[0]
            args = self.parts[1:]
{'fraction_non_alphanumeric': 0.07811877241929055, 'fraction_numerical': 0.0027899561578318055, 'mean_word_length': 1.4627085377821394, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7509997', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1112, 'n_words': 419}