<commit_before># -*- coding: utf-8 -*-

from flask import flash
from flask.ext.babel import gettext, ngettext, lazy_gettext
from sqlalchemy.orm import class_mapper, joinedload
from sqlalchemy.exc import IntegrityError
from sqlalchemy import MetaData
from sqlalchemy import func
from app import db

import sqlalchemy as sa
import sys
from mixins import FileColumn, ImageColumn
from ..filemanager import FileManager, ImageManager


class DataModel():
    obj = None
    
    """ Messages to display on CRUD Events """
    add_row_message = lazy_gettext('Added Row')
    edit_row_message = lazy_gettext('Changed Row')
    delete_row_message = lazy_gettext('Deleted Row')
    delete_integrity_error_message = lazy_gettext('Associated data exists, please delete them first')
    add_integrity_error_message = lazy_gettext('Integrity error, probably unique constraint')
    edit_integrity_error_message = lazy_gettext('Integrity error, probably unique constraint')
    general_error_message = lazy_gettext('General Error')

    def __init__(self, obj):
        self.obj = obj

class SQLAModel(DataModel):
    """
    SQLAModel
    Implements SQLA support methods for views
    """
    def __init__(self, obj):
        DataModel.__init__(self, obj)

    
    def _get_base_query(self, query = None, filters = {}, order_column = '', order_direction = ''):
        for filter_key in filters:
            try:
                rel_model, rel_direction = self._get_related_model(filter_key)
                item = (filters.get(filter_key))
                if rel_direction == 'MANYTOONE':
                    query = query.filter(getattr(self.obj,filter_key) == item)
                elif rel_direction == 'MANYTOMANY':
                    query = query.filter(getattr(self.obj,filter_key).contains(item))
                else:
                    pass
            except:
                if isinstance(self.obj.__mapper__.columns[filter_key].type, sa.types.String):
                    query = query.filter(getattr(self.obj,filter_key).like(filters.get(filter_key) + '%%'))
        if (order_column != ''):
            query = query.order_by(order_column + ' ' + order_direction)
        return query
    
    """
    QUERY
    filters: dict with filters {<col_name>:<value,...}
    order_column
    order_direction: <'asc'|'desc'>
    """
    def query(self, filters = {}, order_column = '', order_direction = ''):
        query = self.obj.query
        query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
        return query.all()


    def query_simple_group(self, group_by = '', filters = {}, order_column = '', order_direction = ''):
        try:
            rel_model, rel_direction = self._get_related_model(group_by)
            query = db.session.query(rel_model,func.count(self.obj.id))
            query = query.join(rel_model, getattr(self.obj,group_by))
            query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
            query = query.group_by(rel_model)
            return query.all()
        except:
            query = db.session.query(rel_model,func.count(self.obj.id))
            query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
            query = query.group_by(group_by)
            return query.all()


    """
    -----------------------------------------
         FUNCTIONS for Testing TYPES
    -----------------------------------------
    """
    def is_image(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, ImageColumn)

    def is_file(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, FileColumn)

    def is_string(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.String)

    def is_text(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Text)

    def is_integer(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Integer)

    def is_boolean(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Boolean)

    def is_date(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Date)

    def is_datetime(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.DateTime)

    def is_relation(self, prop):
        return isinstance(prop, sa.orm.properties.RelationshipProperty)

    def is_relation_col(self, col):
        for i in self.get_properties_iterator():
            if self.is_relation(i):
                if (i.key == col):
                    return self.is_relation(i)
        return False

    def is_relation_many_to_one(self, prop):
        return (prop.direction.name == 'MANYTOONE')

    def is_relation_many_to_many(self, prop):
        return (prop.direction.name == 'MANYTOMANY')

    def is_pk(self, col):
        return col.primary_key

    def is_fk(self, col):
        return  col.foreign_keys

    """
    -----------------------------------------
           FUNCTIONS FOR CRUD OPERATIONS
    -----------------------------------------
    """
    def add(self, item):
        #try:
            db.session.add(item)
            db.session.commit()
            flash(unicode(self.add_row_message),'success')
            return True
        #except IntegrityError as e:
        #    flash(unicode(self.add_integrity_error_message),'warning')
        #    db.session.rollback()
        #    return False
        #except:
        #    flash(unicode(self.general_error_message + ' '  + str(sys.exc_info()[0])),'danger')
        #    db.session.rollback()
        #    return False

    def edit(self, item):
        try:
            db.session.merge(item)
            db.session.commit()
            flash(unicode(self.edit_row_message),'success')
            return True
        except IntegrityError as e:
            flash(unicode(self.edit_integrity_error_message),'warning')
            db.session.rollback()
            return False
        except:
            flash(unicode(self.general_error_message + ' '  + str(sys.exc_info()[0])),'danger')
            db.session.rollback()
            return False
                
    
    def delete(self, item):
        #try:
            self._delete_files(item)
            db.session.delete(item)
            db.session.commit()
            flash(unicode(self.delete_row_message),'success')
            return True
        #except IntegrityError as e:
        #    flash(unicode(self.delete_integrity_error_message),'warning')
        #    db.session.rollback()
        #    return False
        #except:
        #    flash(unicode(self.general_error_message + ' '  + str(sys.exc_info()[0])),'danger')
        #    db.session.rollback()
        #    return False
        
        
    """
    FILE HANDLING METHODS
    """
    def _add_files(self, this_request, item):
        fm = FileManager()
        im = ImageManager()
        for file_col in this_request.files:
            if self.is_file(file_col):                
                fm.save_file(this_request.files[file_col],getattr(item, file_col))
        for file_col in this_request.files:
            if self.is_image(file_col):
                im.save_file(this_request.files[file_col], getattr(item,file_col))
                
        
    def _delete_files(self, item):
        fm = FileManager()
        im = ImageManager()
        for file_col in self.get_file_column_list():
            if self.is_file(file_col):
                if getattr(item,file_col):
                    fm.delete_file(getattr(item, file_col))
        for file_col in self.get_image_column_list():
            if self.is_image(file_col):
                if getattr(item,file_col):
                    im.delete_file(getattr(item,file_col))<commit_msg>Update datamodel.py<commit_after># -*- coding: utf-8 -*-

from flask import flash
from flask.ext.babel import gettext, ngettext, lazy_gettext
from sqlalchemy.orm import class_mapper, joinedload
from sqlalchemy.exc import IntegrityError
from sqlalchemy import MetaData
from sqlalchemy import func
from app import db

import sqlalchemy as sa
import sys
from mixins import FileColumn, ImageColumn
from ..filemanager import FileManager, ImageManager


class DataModel():
    obj = None
    
    """ Messages to display on CRUD Events """
    add_row_message = lazy_gettext('Added Row')
    edit_row_message = lazy_gettext('Changed Row')
    delete_row_message = lazy_gettext('Deleted Row')
    delete_integrity_error_message = lazy_gettext('Associated data exists, please delete them first')
    add_integrity_error_message = lazy_gettext('Integrity error, probably unique constraint')
    edit_integrity_error_message = lazy_gettext('Integrity error, probably unique constraint')
    general_error_message = lazy_gettext('General Error')

    def __init__(self, obj):
        self.obj = obj

class SQLAModel(DataModel):
    """
    SQLAModel
    Implements SQLA support methods for views
    """
    db_session = None
    
    def __init__(self, obj, db_session = None):
        self.db_session = db_session
        DataModel.__init__(self, obj)

    
    def _get_base_query(self, query = None, filters = {}, order_column = '', order_direction = ''):
        for filter_key in filters:
            try:
                rel_model, rel_direction = self._get_related_model(filter_key)
                item = (filters.get(filter_key))
                if rel_direction == 'MANYTOONE':
                    query = query.filter(getattr(self.obj,filter_key) == item)
                elif rel_direction == 'MANYTOMANY':
                    query = query.filter(getattr(self.obj,filter_key).contains(item))
                else:
                    pass
            except:
                if isinstance(self.obj.__mapper__.columns[filter_key].type, sa.types.String):
                    query = query.filter(getattr(self.obj,filter_key).like(filters.get(filter_key) + '%%'))
        if (order_column != ''):
            query = query.order_by(order_column + ' ' + order_direction)
        return query
    
    """
    QUERY
    filters: dict with filters {<col_name>:<value,...}
    order_column
    order_direction: <'asc'|'desc'>
    """
    def query(self, filters = {}, order_column = '', order_direction = ''):
        query = self.obj.query
        query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
        return query.all()


    def query_simple_group(self, group_by = '', filters = {}, order_column = '', order_direction = ''):
        try:
            rel_model, rel_direction = self._get_related_model(group_by)
            query = db.session.query(rel_model,func.count(self.obj.id))
            query = query.join(rel_model, getattr(self.obj,group_by))
            query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
            query = query.group_by(rel_model)
            return query.all()
        except:
            query = db.session.query(rel_model,func.count(self.obj.id))
            query = self._get_base_query(query = query, filters = filters, order_column = order_column, order_direction = order_direction)
            query = query.group_by(group_by)
            return query.all()


    """
    -----------------------------------------
         FUNCTIONS for Testing TYPES
    -----------------------------------------
    """
    def is_image(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, ImageColumn)

    def is_file(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, FileColumn)

    def is_string(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.String)

    def is_text(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Text)

    def is_integer(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Integer)

    def is_boolean(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Boolean)

    def is_date(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.Date)

    def is_datetime(self, col_name):
        return isinstance(self.obj.__mapper__.columns[col_name].type, sa.types.DateTime)

    def is_relation(self, prop):
        return isinstance(prop, sa.orm.properties.RelationshipProperty)

    def is_relation_col(self, col):
        for i in self.get_properties_iterator():
            if self.is_relation(i):
                if (i.key == col):
                    return self.is_relation(i)
        return False

    def is_relation_many_to_one(self, prop):
        return (prop.direction.name == 'MANYTOONE')

    def is_relation_many_to_many(self, prop):
        return (prop.direction.name == 'MANYTOMANY')

    def is_pk(self, col):
        return col.primary_key

    def is_fk(self, col):
        return  col.foreign_keys

    """
    -----------------------------------------
           FUNCTIONS FOR CRUD OPERATIONS
    -----------------------------------------
    """
    def add(self, item):
        try:
            db.session.add(item)
            db.session.commit()
            flash(unicode(self.add_row_message),'success')
            return True
        except IntegrityError as e:
            flash(unicode(self.add_integrity_error_message),'warning')
            db.session.rollback()
            return False
        except:
            flash(unicode(self.general_error_message + ' '  + str(sys.exc_info()[0])),'danger')
            db.session.rollback()
            return False

    def edit(self, item):
        try:
            db.session.merge(item)
            db.session.commit()
            flash(unicode(self.edit_row_message),'success')
            return True
        except IntegrityError as e:
            flash(unicode(self.edit_integrity_error_message),'warning')
            db.session.rollback()
            return False
        except:
            flash(unicode(self.general_error_message + ' '  + str(sys.exc_info()[0])),'danger')
            db.session.rollback()
            return False
                
    
    def delete(self, item):
        try:
            self._delete_files(item)
            db.session.delete(item)
            db.session.commit()
            flash(unicode(self.delete_row_message),'success')
            return True
        except IntegrityError as e:
            flash(unicode(self.delete_integrity_error_message),'warning')
            db.session.rollback()
            return False
        except:
            flash(unicode(self.general_error_message + ' '  + str(sys.exc_info()[0])),'danger')
            db.session.rollback()
            return False
        
        
    """
    FILE HANDLING METHODS
    """
    def _add_files(self, this_request, item):
        fm = FileManager()
        im = ImageManager()
        for file_col in this_request.files:
            if self.is_file(file_col):                
                fm.save_file(this_request.files[file_col],getattr(item, file_col))
        for file_col in this_request.files:
            if self.is_image(file_col):
                im.save_file(this_request.files[file_col], getattr(item,file_col))
                
        
    def _delete_files(self, item):
        fm = FileManager()
        im = ImageManager()
        for file_col in self.get_file_column_list():
            if self.is_file(file_col):
                if getattr(item,file_col):
                    fm.delete_file(getattr(item, file_col))
        for file_col in self.get_image_column_list():
            if self.is_image(file_col):
                if getattr(item,file_col):
                    im.delete_file(getattr(item,file_col))
{'fraction_non_alphanumeric': 0.12281460431210231, 'fraction_numerical': 0.0004942237598072527, 'mean_word_length': 2.2376, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5432941', 'n_tokens_mistral': 4741, 'n_tokens_neox': 4595, 'n_words': 902}