<commit_before>'''
Created on 23 oct. 2015

Updated on 11 may 2015

@author: arxit
'''

import os

import processing
from processing.tools import *

from qgis.core import *
import qgis.utils
from PyQt4.QtGui import QAction
from PyQt4.QtCore import QCoreApplication

from PagLuxembourg.schema import *
import PagLuxembourg.main

from error_summary_dialog import ErrorSummaryDialog

class DataChecker(object):
    '''
    Main class for the data checker widget
    '''

    def __init__(self):
        '''
        Constructor
        '''
    
    def run(self):
        '''
        Runs the widget
        
        :returns: True if there's no errors
        :rtype: Boolean
        '''
        
        project = PagLuxembourg.main.current_project
        
        if not project.isPagProject():
            return
        
        layer_structure_errors = list()
        data_errors = list()
        
        # Iterates through XSD types
        for type in PagLuxembourg.main.xsd_schema.types:
            layer = project.getLayer(type)
            
            if layer is None:
                continue
            
            warn_errors, fatal_errors = self.checkLayerStructure(layer, type)
            layer_structure_errors = layer_structure_errors + warn_errors + fatal_errors
            
            if len(fatal_errors)>0:
                continue
            
            layer_data_errors = self.checkLayerData(layer, type)
            data_errors.append(layer_data_errors)
        
        # Flatten data errors
        data_errors_flat = list()
        for layer, errors in data_errors:
            for feature, field, message in errors:
                data_errors_flat.append((layer, feature, field, message))
        
        valid = (len(layer_structure_errors) + len(data_errors_flat)) == 0
        
        if valid:
            PagLuxembourg.main.qgis_interface.messageBar().clearWidgets()
            PagLuxembourg.main.qgis_interface.messageBar().pushSuccess(QCoreApplication.translate('DataChecker','Success'),
                                                                       QCoreApplication.translate('DataChecker','No errors found.'))
        else:
            self.dlg = ErrorSummaryDialog(layer_structure_errors, data_errors)
            self.dlg.show()
        
        return valid 
        
    # Datatype mapping allowed while checking. For a given XSD type, several QGIS type may be allowed or compatible
    XSD_QGIS_ALLOWED_DATATYPE_MAP = [(DataType.STRING, QVariant.String),
                                     (DataType.STRING, QVariant.LongLong),
                                     (DataType.STRING, QVariant.Int),
                                     (DataType.STRING, QVariant.Double),
                                     (DataType.INTEGER, QVariant.LongLong),
                                     (DataType.INTEGER, QVariant.Int),
                                     (DataType.DOUBLE, QVariant.Double),
                                     (DataType.DOUBLE, QVariant.LongLong),
                                     (DataType.DOUBLE, QVariant.Int),
                                     (DataType.DATE, QVariant.String)]
    
    def checkLayerStructure(self, layer, xsd_type):
        '''
        Checks a layer structure against the XSD type
        Missing field, data type mismatch
        
        :param layer: The vector layer to check
        :type layer: QgsVectorLayer
        
        :param type: XSD schema type
        :type type: PAGType
        
        :returns: A list of warning and fatal error
        :rtype: Tuple : warning, fatal. Layer (QgsVectorLayer), field (PAGField), message (str, QString)
        '''
        
        layer_fields = layer.dataProvider().fields()
        warn_errors = list()
        fatal_errors = list()
        
        # Check geometry type
        if xsd_type.geometry_type is not None and XSD_QGIS_GEOMETRYTYPE_MAP[xsd_type.geometry_type] != layer.geometryType():
            fatal_errors.append((layer, None, QCoreApplication.translate('DataChecker','Geometry type mismatch, expected : {}').format(xsd_type.geometry_type)))
        
        # Check field structure
        for field in xsd_type.fields:
            # Check field missing
            layer_index = layer_fields.fieldNameIndex(field.name)
            if layer_index == -1:
                if field.nullable:
                    warn_errors.append((layer, field, QCoreApplication.translate('DataChecker','Nullable field is missing')))
                else:
                    warn_errors.append((layer, field, QCoreApplication.translate('DataChecker','Non nullable field is missing')))
                
                continue
            
            # Check field datatype
            layer_field = layer_fields[layer_index]
            found = False
            for xsd_type, qgis_type in self.XSD_QGIS_ALLOWED_DATATYPE_MAP:
                if layer_field.type() == qgis_type and field.type == xsd_type:
                    found = True
                    break
            
            if not found:
                fatal_errors.append((layer, field, QCoreApplication.translate('DataChecker','Field datatype mismatch, expected : {}').format(field.type)))
        
        return warn_errors, fatal_errors
    
    def checkLayerData(self, layer, xsd_type):
        '''
        Checks the data of a layer against the XSD type
        
        :param layer: The vector layer to check
        :type layer: QgsVectorLayer
        
        :param type: XSD schema type
        :type type: PAGType
        
        :returns: A list of data error
        :rtype: Tuples : Layer (QgsVectorLayer), list of tuple Feature (QgsFeature), field (PAGField), message (str, QString)
        '''
        
        errors = list()
        
        # Layer definition        
        layer_PAG=PagLuxembourg.main.current_project.getLayer(PagLuxembourg.main.xsd_schema.getTypeFromTableName('PAG.MODIFICATION_PAG'))
        layer_ZONAGE=PagLuxembourg.main.current_project.getLayer(PagLuxembourg.main.xsd_schema.getTypeFromTableName('PAG.CONST_A_CONS_POINT'))
        # Selection definition
        layer_PAG.selectedFeatures()
        
        # Selection by intersection with 'MODIFICATION PAG' layer
        if layer.name() != 'MODIFICATION PAG' :
            qgis.utils.iface.messageBar().pushMessage("HEY2")
            areas = []
            for PAG_feature in layer_PAG.getFeatures():
                #qgis.utils.iface.messageBar().pushMessage("HEY")
                cands = layer.getFeatures()
                for ZONAGE_feature in cands:
                    if PAG_feature.geometry().intersects(ZONAGE_feature.geometry()):
                        areas.append(ZONAGE_feature.id())

            layer.select(areas)
        
        
        for feature in layer.dataProvider().getFeatures():
            errors += self.checkFeatureData(feature, xsd_type)
        
        return layer, errors
    
    def checkFeatureData(self, feature, xsd_type):
        '''
        Checks the data of a feature against the XSD type
        
        :param feature: The feature to check
        :type feature: QgsFeature
        
        :param type: XSD schema type
        :type type: PAGType
        
        :returns: A list of  error<commit_msg>Intersections MODIFICATION PAG avec les autres layers<commit_after>'''
Created on 23 oct. 2015

Updated on 11 may 2015

@author: arxit
'''

import os

import processing
from processing.tools import *

from qgis.core import *
import qgis.utils
from PyQt4.QtGui import QAction
from PyQt4.QtCore import QCoreApplication

from PagLuxembourg.schema import *
import PagLuxembourg.main

from error_summary_dialog import ErrorSummaryDialog

class DataChecker(object):
    '''
    Main class for the data checker widget
    '''

    def __init__(self):
        '''
        Constructor
        '''
    
    def run(self):
        '''
        Runs the widget
        
        :returns: True if there's no errors
        :rtype: Boolean
        '''
        
        project = PagLuxembourg.main.current_project
        
        if not project.isPagProject():
            return
        
        layer_structure_errors = list()
        data_errors = list()
        
        # Iterates through XSD types
        for type in PagLuxembourg.main.xsd_schema.types:
            layer = project.getLayer(type)
            
            if layer is None:
                continue
            
            warn_errors, fatal_errors = self.checkLayerStructure(layer, type)
            layer_structure_errors = layer_structure_errors + warn_errors + fatal_errors
            
            if len(fatal_errors)>0:
                continue
            
            layer_data_errors = self.checkLayerData(layer, type)
            data_errors.append(layer_data_errors)
        
        # Flatten data errors
        data_errors_flat = list()
        for layer, errors in data_errors:
            for feature, field, message in errors:
                data_errors_flat.append((layer, feature, field, message))
        
        valid = (len(layer_structure_errors) + len(data_errors_flat)) == 0
        
        if valid:
            PagLuxembourg.main.qgis_interface.messageBar().clearWidgets()
            PagLuxembourg.main.qgis_interface.messageBar().pushSuccess(QCoreApplication.translate('DataChecker','Success'),
                                                                       QCoreApplication.translate('DataChecker','No errors found.'))
        else:
            self.dlg = ErrorSummaryDialog(layer_structure_errors, data_errors)
            self.dlg.show()
        
        return valid 
        
    # Datatype mapping allowed while checking. For a given XSD type, several QGIS type may be allowed or compatible
    XSD_QGIS_ALLOWED_DATATYPE_MAP = [(DataType.STRING, QVariant.String),
                                     (DataType.STRING, QVariant.LongLong),
                                     (DataType.STRING, QVariant.Int),
                                     (DataType.STRING, QVariant.Double),
                                     (DataType.INTEGER, QVariant.LongLong),
                                     (DataType.INTEGER, QVariant.Int),
                                     (DataType.DOUBLE, QVariant.Double),
                                     (DataType.DOUBLE, QVariant.LongLong),
                                     (DataType.DOUBLE, QVariant.Int),
                                     (DataType.DATE, QVariant.String)]
    
    def checkLayerStructure(self, layer, xsd_type):
        '''
        Checks a layer structure against the XSD type
        Missing field, data type mismatch
        
        :param layer: The vector layer to check
        :type layer: QgsVectorLayer
        
        :param type: XSD schema type
        :type type: PAGType
        
        :returns: A list of warning and fatal error
        :rtype: Tuple : warning, fatal. Layer (QgsVectorLayer), field (PAGField), message (str, QString)
        '''
        
        layer_fields = layer.dataProvider().fields()
        warn_errors = list()
        fatal_errors = list()
        
        # Check geometry type
        if xsd_type.geometry_type is not None and XSD_QGIS_GEOMETRYTYPE_MAP[xsd_type.geometry_type] != layer.geometryType():
            fatal_errors.append((layer, None, QCoreApplication.translate('DataChecker','Geometry type mismatch, expected : {}').format(xsd_type.geometry_type)))
        
        # Check field structure
        for field in xsd_type.fields:
            # Check field missing
            layer_index = layer_fields.fieldNameIndex(field.name)
            if layer_index == -1:
                if field.nullable:
                    warn_errors.append((layer, field, QCoreApplication.translate('DataChecker','Nullable field is missing')))
                else:
                    warn_errors.append((layer, field, QCoreApplication.translate('DataChecker','Non nullable field is missing')))
                
                continue
            
            # Check field datatype
            layer_field = layer_fields[layer_index]
            found = False
            for xsd_type, qgis_type in self.XSD_QGIS_ALLOWED_DATATYPE_MAP:
                if layer_field.type() == qgis_type and field.type == xsd_type:
                    found = True
                    break
            
            if not found:
                fatal_errors.append((layer, field, QCoreApplication.translate('DataChecker','Field datatype mismatch, expected : {}').format(field.type)))
        
        return warn_errors, fatal_errors
    
    def checkLayerData(self, layer, xsd_type):
        '''
        Checks the data of a layer against the XSD type
        
        :param layer: The vector layer to check
        :type layer: QgsVectorLayer
        
        :param type: XSD schema type
        :type type: PAGType
        
        :returns: A list of data error
        :rtype: Tuples : Layer (QgsVectorLayer), list of tuple Feature (QgsFeature), field (PAGField), message (str, QString)
        '''
        
        errors = list()
        
        # Layer definition        
        layer_PAG = PagLuxembourg.main.current_project.getLayer(PagLuxembourg.main.xsd_schema.getTypeFromTableName('PAG.MODIFICATION_PAG'))
                
        # Selection definition
        #selection_PAG = layer_PAG.selectedFeatures()
        
        # Selection by intersection with 'MODIFICATION PAG' layer
        if layer.name() != 'MODIFICATION PAG' :
            qgis.utils.iface.messageBar().pushMessage("HEY2")
            areas = []
            for PAG_feature in layer_PAG.getFeatures():
                #qgis.utils.iface.messageBar().pushMessage("HEY")
                cands = layer.getFeatures()
                for layer_features in cands:
                    if PAG_feature.geometry().intersects(layer_features.geometry()):
                        areas.append(layer_features.id())

            layer.select(areas)
        
        
        for feature in layer.dataProvider().getFeatures():
            errors += self.checkFeatureData(feature, xsd_type)
        
        return layer, errors
    
    def checkFeatureData(self, feature, xsd_type):
        '''
        Checks the data of a feature against the XSD type
        
        :param feature: The feature to check
        :type feature: QgsFeature
        
        :param type: XSD schema type
        :type type: PAGType
        
        :returns: A list of  error
{'fraction_non_alphanumeric': 0.07820998278829604, 'fraction_numerical': 0.002478485370051635, 'mean_word_length': 1.7626473944465577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1770148', 'n_tokens_mistral': 3831, 'n_tokens_neox': 3756, 'n_words': 932}