<commit_before># coding: utf-8
from normality import slugify
from sqlalchemy import and_
# from sqlalchemy.sql.expression import extract
from sqlalchemy.sql.expression import select

LABEL_SEP = u'‽'


def valid_name(name):
    slug = slugify(name, sep='_')
    if slug != name or slug == 'row_count':
        raise ValueError('Invalid identifier: %s' % name)
    return slug


def resolve_columns(model, path):
    part = None
    if ':' in path:
        path, part = path.rsplit(':', 1)
    for attribute in model.match_qualified(path):
        # TODO: allow extracting from date objects
        label = attribute.path.replace('.', LABEL_SEP)
        yield attribute, attribute.column.label(label)


def resolve_column(model, path):
    columns = list(resolve_columns(model, path))
    for attr, column in columns:
        if len(columns) == 1 or attr.key:
            return column


def make_columns(project, paths):
    cols = []
    for path in paths:
        for attr, col in resolve_columns(project.model, path):
            cols.append(col)
    return cols


def make_filters(project, filters):
    filter = and_()
    for path, value in filters:
        col = resolve_column(project.model, path)<commit_msg>extract years etc from a field<commit_after># coding: utf-8
from normality import slugify
from sqlalchemy import and_
from sqlalchemy.sql.expression import extract
from sqlalchemy.sql.expression import select

LABEL_SEP = u'‽'


def valid_name(name):
    slug = slugify(name, sep='_')
    if slug != name or slug == 'row_count':
        raise ValueError('Invalid identifier: %s' % name)
    return slug


def resolve_columns(model, path):
    part = None
    if ':' in path:
        path, part = path.rsplit(':', 1)
    for attribute in model.match_qualified(path):
        # TODO: allow extracting from date objects
        label = attribute.path.replace('.', LABEL_SEP)
        if part is not None:
            label = '%s:%s' % (label, part)
            column = extract(part, attribute.column)
        else:
            column = attribute.column
        yield attribute, column.label(label)


def resolve_column(model, path):
    columns = list(resolve_columns(model, path))
    for attr, column in columns:
        if len(columns) == 1 or attr and attr.key:
            return column


def make_columns(project, paths):
    cols = []
    for path in paths:
        for attr, col in resolve_columns(project.model, path):
            cols.append(col)
    return cols


def make_filters(project, filters):
    filter = and_()
    for path, value in filters:
        col = resolve_column(project.model, path)
{'fraction_non_alphanumeric': 0.10243716679360244, 'fraction_numerical': 0.002284843869002285, 'mean_word_length': 2.8918518518518517, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5359682', 'n_tokens_mistral': 807, 'n_tokens_neox': 767, 'n_words': 251}