<commit_before># I'm not sure how to separate this from the rest of my project
from ..models import Base as DeclarativeBase
from sqlalchemy import (
    Column,
    Integer,
    Text,
    String,
    Float,
    Date,
    Boolean,
    Interval,
    Numeric,
    
    ForeignKey,
)

import json

def build_field_getter(db_class):
    def f(key):
        if type(key) == str:
            return getattr(db_class, key)
        return key
    return f

class ConcisionSource(object):
    """
    The base object for database objects looked at by Concision
    """
    def __init__(self, db_class, **kwargs):
        super(ConcisionSource, self).__init__()
        self.db_class = db_class
        
        self.name = kwargs.get('name', db_class.__tablename__)
        self.label = kwargs.get('label', db_class.__tablename__)
        
        self.query_as_single = kwargs.get('query_as_single', True)
        self.query_as_aggregate = kwargs.get('query_as_aggregate', True)
        self.column_labels = kwargs.get('column_labels', {})
        self.column_converters = kwargs.get('column_converters', {})
        
        self.aliases = kwargs.get('aliases', {})
        
        if "keys" in kwargs:
            # self.keys = tuple(map(build_field_getter(db_class), kwargs['keys']))
            self.keys = kwargs['keys']
        else:
            raise Exception("Cannot auto-generate keys yet")
        
        # TODO - Document the multiple ways to pull columns
        if "columns" in kwargs:
            self.columns = kwargs['columns']
        elif "columns_from_labels" in kwargs and len(self.column_labels) > 0:
            self.columns = tuple(self.column_labels.keys())
        else:
            ignored_columns = kwargs.get('ignored_columns', [])
            self.columns = [c.name for c in db_class.__table__.columns if c not in ignored_columns]

class StoredQuery(DeclarativeBase):
    """Used to store a query in the database, it is then converted into
    an executable query when needed."""
    
    __tablename__ = "concision_queries"
    id       = Column(Integer, primary_key=True)
    name     = Column(String, nullable=False)
    data     = Column(Text, nullable=False)
    creator  = Column(Integer, ForeignKey("users.id"), nullable=False)
    complete = Column(Boolean, nullable=False, default=False)
    
    def extract_data(self):
        self.jdata = json.loads(self.data)
        return self.jdata
    
    def compress_data(self, d=None):
        if d == None:
            self.data = json.dumps(self.jdata)
        else:
            self.data = json.dumps(d)
        return self.data
<commit_msg>Columns default to being sorted<commit_after># I'm not sure how to separate this from the rest of my project
from ..models import Base as DeclarativeBase
from sqlalchemy import (
    Column,
    Integer,
    Text,
    String,
    Float,
    Date,
    Boolean,
    Interval,
    Numeric,
    
    ForeignKey,
)

import json

def build_field_getter(db_class):
    def f(key):
        if type(key) == str:
            return getattr(db_class, key)
        return key
    return f

class ConcisionSource(object):
    """
    The base object for database objects looked at by Concision
    """
    def __init__(self, db_class, **kwargs):
        super(ConcisionSource, self).__init__()
        self.db_class = db_class
        
        self.name = kwargs.get('name', db_class.__tablename__)
        self.label = kwargs.get('label', db_class.__tablename__)
        
        self.query_as_single = kwargs.get('query_as_single', True)
        self.query_as_aggregate = kwargs.get('query_as_aggregate', True)
        self.column_labels = kwargs.get('column_labels', {})
        self.column_converters = kwargs.get('column_converters', {})
        
        self.aliases = kwargs.get('aliases', {})
        
        if "keys" in kwargs:
            # self.keys = tuple(map(build_field_getter(db_class), kwargs['keys']))
            self.keys = kwargs['keys']
        else:
            raise Exception("Cannot auto-generate keys yet")
        
        # TODO - Document the multiple ways to pull columns
        if "columns" in kwargs:
            self.columns = kwargs['columns']
        elif "columns_from_labels" in kwargs and len(self.column_labels) > 0:
            self.columns = list(self.column_labels.keys())
            self.columns.sort()
        else:
            ignored_columns = kwargs.get('ignored_columns', [])
            self.columns = [c.name for c in db_class.__table__.columns if c not in ignored_columns]
            self.columns.sort()

class StoredQuery(DeclarativeBase):
    """Used to store a query in the database, it is then converted into
    an executable query when needed."""
    
    __tablename__ = "concision_queries"
    id       = Column(Integer, primary_key=True)
    name     = Column(String, nullable=False)
    data     = Column(Text, nullable=False)
    creator  = Column(Integer, ForeignKey("users.id"), nullable=False)
    complete = Column(Boolean, nullable=False, default=False)
    
    def extract_data(self):
        self.jdata = json.loads(self.data)
        return self.jdata
    
    def compress_data(self, d=None):
        if d == None:
            self.data = json.dumps(self.jdata)
        else:
            self.data = json.dumps(d)
        return self.data

{'fraction_non_alphanumeric': 0.10324760653275765, 'fraction_numerical': 0.00037544584193730055, 'mean_word_length': 2.4088291746641075, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3537658', 'n_tokens_mistral': 1581, 'n_tokens_neox': 1606, 'n_words': 400}