<commit_before># -*- coding: utf-8 -*-

from flask import flash
from flask.ext.babel import gettext, ngettext, lazy_gettext
from sqlalchemy.orm import class_mapper, joinedload
from sqlalchemy.exc import IntegrityError
from sqlalchemy import MetaData
from sqlalchemy import func

import sqlalchemy as sa
import sys
from mixins import FileColumn, ImageColumn
from ..filemanager import FileManager, ImageManager


class DataModel():
    obj = None
    
    """ Messages to display on CRUD Events """
    add_row_message = lazy_gettext('Added Row')
    edit_row_message = lazy_gettext('Changed Row')
    delete_row_message = lazy_gettext('Deleted Row')
    delete_integrity_error_message = lazy_gettext('Associated data exists, please delete them first')
    add_integrity_error_message = lazy_gettext('Integrity error, probably unique constraint')
    edit_integrity_error_message = lazy_gettext('Integrity error, probably unique constraint')
    general_error_message = lazy_gettext('General Error')

    def __init__(self, obj):
        self.obj = obj

class SQLAModel(DataModel):
    """
    SQLAModel
    Implements SQLA support methods for views
    """
    session = None
    
    def __init__(self, obj, session = None):
        self.session = session
        DataModel.__init__(self, obj)

    
    def _get_base_query(self, query = None, filters = {}, order_column = '', order_direction = ''):
        for filter_key in filters:
            try:
                rel_model, rel_direction = self._get_related_model(filter_key)
                item = (filters.get(filter_key))
                if rel_direction == 'MANYTOONE':
                    query = query.filter(getattr(self.obj,filter_key) == item)
                elif rel_direction == 'MANYTOMANY':
                    query = query.filter(getattr(self.obj,filter_key).contains(item))
                else:
                    pass
            except:
                if self.is_string(filter_key) or self.is_text(filter_key):
                    query = query.filter(getattr(self.obj,filter_key).like(filters.get(filter_key) + '%'))
        if (order_column != ''):
            query = query.order_by(order_column + ' ' + order_direction)
        
        return query

   
    """
    QUERY
    filters: dict with filters {<col_name>:<value,...}
    order_column
    order_direction: <'asc'|'desc'>
    """
    def query(self, filters = {}, order_column = '', order_direction = '',
                page = None, page_size = None):
        query = self.session.query(self.obj)
        query_count = self.session.query(func.count('*')).select_from(self.obj)
        
        query_count = self._get_base_query(query = query_count, 
                        filters = filters, 
                        order_column = order_column, 
                        order_direction = order_direction)
        
        query = self._get_base_query(query = query, 
                        filters = filters, 
                        order_column = order_column, 
                        order_direction = order_direction)
        
        count = query_count.scalar() 
        
        if page: 
            print "OFFSET", page*page_size
            query = query.offset(page*page_size)
        if page_size:
            query = query.limit(page_size)
        
        return count, query.all()


    def query_simple_group(self, group_by = '', filters = {}, order_column = '', order_direction = ''):
        try:
            rel_model, rel_direction = self._get_related_model(group_by)
            query = self.session.query(rel_model,func.count(self.obj.id))
            query = query.join(rel_model, getattr(self.obj,group_by))
            query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
            query = query.group_by(rel_model)
            return query.all()
        except:
            query = self.session.query(rel_model,func.count(self.obj.id))
            query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
            query = query.group_by(group_by)
            return query.all()

    def query_month_group(self, group_by = '', filters = {}, order_column = '', order_direction = ''):
        query = self.session.query(self.obj)
        query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
        query = query.group_by(func.month(getattr(self.obj,group_by)))
        return query.all()
        


    """
    -----------------------------------------
         FUNCTIONS for Testing TYPES
    -----------------------------------------
    """
    def is_image(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, ImageColumn)

    def is_file(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, FileColumn)

    def is_string(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.String)

    def is_text(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Text)

    def is_integer(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Integer)

    def is_boolean(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Boolean)

    def is_date(self, col_name):<commit_msg>Update datamodel.py<commit_after># -*- coding: utf-8 -*-

from flask import flash
from flask.ext.babel import gettext, ngettext, lazy_gettext
from sqlalchemy.orm import class_mapper, joinedload
from sqlalchemy.exc import IntegrityError
from sqlalchemy import MetaData
from sqlalchemy import func

import sqlalchemy as sa
import sys
from mixins import FileColumn, ImageColumn
from ..filemanager import FileManager, ImageManager


class DataModel():
    obj = None
    
    """ Messages to display on CRUD Events """
    add_row_message = lazy_gettext('Added Row')
    edit_row_message = lazy_gettext('Changed Row')
    delete_row_message = lazy_gettext('Deleted Row')
    delete_integrity_error_message = lazy_gettext('Associated data exists, please delete them first')
    add_integrity_error_message = lazy_gettext('Integrity error, probably unique constraint')
    edit_integrity_error_message = lazy_gettext('Integrity error, probably unique constraint')
    general_error_message = lazy_gettext('General Error')

    def __init__(self, obj):
        self.obj = obj

class SQLAModel(DataModel):
    """
    SQLAModel
    Implements SQLA support methods for views
    """
    session = None
    
    def __init__(self, obj, session = None):
        self.session = session
        DataModel.__init__(self, obj)

    
    def _get_base_query(self, query = None, filters = {}, order_column = '', order_direction = ''):
        for filter_key in filters:
            try:
                rel_model, rel_direction = self._get_related_model(filter_key)
                item = (filters.get(filter_key))
                if rel_direction == 'MANYTOONE':
                    query = query.filter(getattr(self.obj,filter_key) == item)
                elif rel_direction == 'MANYTOMANY':
                    query = query.filter(getattr(self.obj,filter_key).contains(item))
                else:
                    pass
            except:
                if self.is_string(filter_key) or self.is_text(filter_key):
                    query = query.filter(getattr(self.obj,filter_key).like(filters.get(filter_key) + '%'))
        if (order_column != ''):
            query = query.order_by(order_column + ' ' + order_direction)
        
        return query

   
    """
    QUERY
    filters: dict with filters {<col_name>:<value,...}
    order_column
    order_direction: <'asc'|'desc'>
    """
    def query(self, filters = {}, order_column = '', order_direction = '',
                page = None, page_size = None):
        query = self.session.query(self.obj)
        query_count = self.session.query(func.count('*')).select_from(self.obj)
        
        query_count = self._get_base_query(query = query_count, 
                        filters = filters, 
                        order_column = order_column, 
                        order_direction = order_direction)
        
        query = self._get_base_query(query = query, 
                        filters = filters, 
                        order_column = order_column, 
                        order_direction = order_direction)
        
        count = query_count.scalar() 
        
        if page: 
            print "OFFSET", page*page_size
            query = query.offset(page*page_size)
        if page_size:
            query = query.limit(page_size)
        
        return count, query.all()


    def query_simple_group(self, group_by = '', filters = {}, order_column = '', order_direction = ''):
        try:
            rel_model, rel_direction = self._get_related_model(group_by)
            query = self.session.query(rel_model,func.count(self.obj.id))
            query = query.join(rel_model, getattr(self.obj,group_by))
            query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
            query = query.group_by(rel_model)
            return query.all()
        except:
            query = self.session.query(rel_model,func.count(self.obj.id))
            query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
            query = query.group_by(group_by)
            return query.all()

    def query_month_group(self, group_by = '', filters = {}, order_column = '', order_direction = ''):
        query = self.session.query(func.month(getattr(self.obj,group_by)), func.count(self.obj.id))
        query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
        query = query.group_by(func.month(getattr(self.obj,group_by)))
        return query.all()
        


    """
    -----------------------------------------
         FUNCTIONS for Testing TYPES
    -----------------------------------------
    """
    def is_image(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, ImageColumn)

    def is_file(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, FileColumn)

    def is_string(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.String)

    def is_text(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Text)

    def is_integer(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Integer)

    def is_boolean(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Boolean)

    def is_date(self, col_name):
{'fraction_non_alphanumeric': 0.11548532731376975, 'fraction_numerical': 0.00018058690744920993, 'mean_word_length': 2.4143033292231815, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6702228', 'n_tokens_mistral': 3184, 'n_tokens_neox': 3086, 'n_words': 643}