<commit_before>import ast
from itertools import groupby
import parallel
import screen
import util
import time
try:
    from collections import OrderedDict
except ImportError:
    # python < 2.7 compatability
    from compat.OrderedDict import OrderedDict
try:
    from collections import Counter
except ImportError:
    # python < 2.7 compatability
    from compat.Counter import Counter

DEBUG = False
__is_aggregate = False

def do(stmt, data):
    global  __is_aggregate
    __is_aggregate = False # reset

    d = data
    if stmt.where:
        d = _where(stmt.where, d)
    if stmt.groupby:
        groups = []
        for k, g in groupby(_groupby(stmt.groupby, d), lambda x: x[stmt.groupby[0]]):
            groups.append(list(g))
        resp = []
        for group in groups:
            resp.append(_fields(stmt.fields, group))
        d = _flatten(resp)
    else:
        d = _fields(stmt.fields, d)
    if stmt.orderby:
        if stmt.orderby[1] == 'desc':
            d = reversed(_orderby(stmt.orderby[0], d))
        else:<commit_msg>Fixed iterator->list bug<commit_after>import ast
from itertools import groupby
import parallel
import screen
import util
import time
try:
    from collections import OrderedDict
except ImportError:
    # python < 2.7 compatability
    from compat.OrderedDict import OrderedDict
try:
    from collections import Counter
except ImportError:
    # python < 2.7 compatability
    from compat.Counter import Counter

DEBUG = False
__is_aggregate = False

def do(stmt, data):
    global  __is_aggregate
    __is_aggregate = False # reset

    d = data
    if stmt.where:
        d = _where(stmt.where, d)
    if stmt.groupby:
        groups = []
        for k, g in groupby(_groupby(stmt.groupby, d), lambda x: x[stmt.groupby[0]]):
            groups.append(list(g))
        resp = []
        for group in groups:
            resp.append(_fields(stmt.fields, group))
        d = _flatten(resp)
    else:
        d = _fields(stmt.fields, d)
    if stmt.orderby:
        if stmt.orderby[1] == 'desc':
            d = list(reversed(_orderby(stmt.orderby[0], d)))
        else:
{'fraction_non_alphanumeric': 0.09258384506376949, 'fraction_numerical': 0.006613131790269249, 'mean_word_length': 2.5898305084745763, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '187956', 'n_tokens_mistral': 672, 'n_tokens_neox': 655, 'n_words': 193}