<commit_before>#!/usr/bin/env python

#===============================================================================
#    Copyright 2017 Geoscience Australia
# 
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
# 
#        http://www.apache.org/licenses/LICENSE-2.0
# 
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#===============================================================================
'''
NetCDFConverter abstract base class for converting data to netCDF

Created on 28Mar.2018

@author: <NAME>
'''
import abc
import logging
import netCDF4
import numpy as np
import osgeo
from collections import OrderedDict

from geophys_utils import get_spatial_ref_from_wkt

class NetCDFVariable(object):
    '''
    Class to manage netCDF variable contents
    '''
    # Define single default chunk size for all dimensions
    DEFAULT_CHUNK_SIZE = 1024
    
    # Define default variable parameters
    DEFAULT_VARIABLE_PARAMETERS = {'complevel': 4, 
                                   'zlib': True, 
                                   'fletcher32': True,
                                   'shuffle': True,
                                   'endian': 'little',
                                   }
    
    def __init__(self, 
                 short_name, 
                 data, dimensions, 
                 fill_value, 
                 attributes, 
                 dtype=None,
                 chunk_size=None, # None means take default, zero means not chunked
                 variable_parameters=None
                 ): 
        '''
        Constructor for class NetCDFVariable to manage netCDF variable contents
        @param variable_parameters: dict containing parameters for netCDF variable creation
        @param dtype: Optional datatype to override data
        @param chunk_size: single default chunk size for all dimensions. None means take default, zero means not chunked.
            Note that custom chunk sizes per dimension can be specified using chunksizes value in variable_parameters
        '''
        self.short_name = short_name # String used for variable name
        self.data = data # Numpy array
        self.dimensions = dimensions # List of <dimension_name> strings for array, or None or empty list for scalar
        self.attributes = attributes # dict of variable attribute <key>:<value> pairs 
        self.variable_parameters = dict(variable_parameters or NetCDFVariable.DEFAULT_VARIABLE_PARAMETERS)
        
        # chunk_size = None means take default, zero means not chunked
        self.chunk_size = chunk_size
        if self.chunk_size is None:
            self.chunk_size = NetCDFVariable.DEFAULT_CHUNK_SIZE

        #TODO: Implement something better for determining the type of scalars
        self.dtype = dtype or (data.dtype if (type(data) == np.ndarray) else 'float64')
        
        if self.dimensions:
            # Set fill value parameter if not already defined
            if fill_value is not None and not self.variable_parameters.get('fill_value'):
                self.variable_parameters['fill_value'] = fill_value
                
            # Set up simple chunking if no 'chunksizes' parameter already defined
            if self.chunk_size and not self.variable_parameters.get('chunksizes'):
                self.variable_parameters['chunksizes'] = [self.chunk_size
                                                     for _dimension_name in self.dimensions
                                                     ]
                
                
                
        
    def create_var_in_dataset(self, nc_output_dataset):
        '''
        Function to create netCDF variable in specified dataset
        '''
        variable_parameters = dict(self.variable_parameters) # Copy this to avoid modifying original
        
        # If not a scalar, check array shape against specified dimensions in netCDF dataset
        if self.dimensions:
            assert set(self.dimensions) <= set(list(nc_output_dataset.dimensions.keys())), 'Invalid dimension(s) specified'

            assert self.data.shape == tuple([nc_output_dataset.dimensions[dimension_name].size
                                             for dimension_name in self.dimensions
                                             ]), 'Invalid array shape for specified dimension(s)'
                                             
            # Ensure that chunk sizes do not exceed array dimensions
            if variable_parameters.get('chunksizes'):
                variable_parameters['chunksizes'] = [min(nc_output_dataset.dimensions[self.dimensions[dimension_index]].size,
                                                              variable_parameters['chunksizes'][dimension_index]
                                                              )
                                                          for dimension_index in range(len(self.dimensions))
                                                          ]
                

                                             
                                             
        output_variable = nc_output_dataset.createVariable(self.short_name,
                                                           self.dtype,
                                                           self.dimensions,
                                                           **variable_parameters
                                                           )
        # Set data values
        output_variable[:] = self.data
        
        # Set variable attributes
        output_variable.setncatts(self.attributes)
        
        return output_variable
<commit_msg>Debugging output improved for data shape checking<commit_after>#!/usr/bin/env python

#===============================================================================
#    Copyright 2017 Geoscience Australia
# 
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
# 
#        http://www.apache.org/licenses/LICENSE-2.0
# 
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#===============================================================================
'''
NetCDFConverter abstract base class for converting data to netCDF

Created on 28Mar.2018

@author: <NAME>
'''
import abc
import logging
import netCDF4
import numpy as np
import osgeo
from collections import OrderedDict

from geophys_utils import get_spatial_ref_from_wkt

class NetCDFVariable(object):
    '''
    Class to manage netCDF variable contents
    '''
    # Define single default chunk size for all dimensions
    DEFAULT_CHUNK_SIZE = 1024
    
    # Define default variable parameters
    DEFAULT_VARIABLE_PARAMETERS = {'complevel': 4, 
                                   'zlib': True, 
                                   'fletcher32': True,
                                   'shuffle': True,
                                   'endian': 'little',
                                   }
    
    def __init__(self, 
                 short_name, 
                 data, 
                 dimensions, 
                 fill_value, 
                 attributes, 
                 dtype=None,
                 chunk_size=None, # None means take default, zero means not chunked
                 variable_parameters=None
                 ): 
        '''
        Constructor for class NetCDFVariable to manage netCDF variable contents
        @param variable_parameters: dict containing parameters for netCDF variable creation
        @param dtype: Optional datatype to override data
        @param chunk_size: single default chunk size for all dimensions. None means take default, zero means not chunked.
            Note that custom chunk sizes per dimension can be specified using chunksizes value in variable_parameters
        '''
        self.short_name = short_name # String used for variable name
        self.data = data # Numpy array
        self.dimensions = dimensions # List of <dimension_name> strings for array, or None or empty list for scalar
        self.attributes = attributes # dict of variable attribute <key>:<value> pairs 
        self.variable_parameters = dict(variable_parameters or NetCDFVariable.DEFAULT_VARIABLE_PARAMETERS)
        
        # chunk_size = None means take default, zero means not chunked
        self.chunk_size = chunk_size
        if self.chunk_size is None:
            self.chunk_size = NetCDFVariable.DEFAULT_CHUNK_SIZE

        #TODO: Implement something better for determining the type of scalars
        self.dtype = dtype or (data.dtype if (type(data) == np.ndarray) else 'float64')
        
        if self.dimensions:
            # Set fill value parameter if not already defined
            if fill_value is not None and not self.variable_parameters.get('fill_value'):
                self.variable_parameters['fill_value'] = fill_value
                
            # Set up simple chunking if no 'chunksizes' parameter already defined
            if self.chunk_size and not self.variable_parameters.get('chunksizes'):
                self.variable_parameters['chunksizes'] = [self.chunk_size
                                                     for _dimension_name in self.dimensions
                                                     ]
                
                
                
        
    def create_var_in_dataset(self, nc_output_dataset):
        '''
        Function to create netCDF variable in specified dataset
        '''
        variable_parameters = dict(self.variable_parameters) # Copy this to avoid modifying original
        
        # If not a scalar, check array shape against specified dimensions in netCDF dataset
        if self.dimensions:
            assert set(self.dimensions) <= set(list(nc_output_dataset.dimensions.keys())), 'Invalid dimension(s) specified'

            expected_shape = tuple([nc_output_dataset.dimensions[dimension_name].size
                                    for dimension_name in self.dimensions
                                    ])
            
            assert self.data.shape == expected_shape, 'Invalid array shape for specified dimension(s). Expected {}, got {}.'.format(expected_shape,
                                                                                                                                    self.data.shape
                                                                                                                                    )
                                             
            # Ensure that chunk sizes do not exceed array dimensions
            if variable_parameters.get('chunksizes'):
                variable_parameters['chunksizes'] = [min(nc_output_dataset.dimensions[self.dimensions[dimension_index]].size,
                                                              variable_parameters['chunksizes'][dimension_index]
                                                              )
                                                          for dimension_index in range(len(self.dimensions))
                                                          ]
                

                                             
                                             
        output_variable = nc_output_dataset.createVariable(self.short_name,
                                                           self.dtype,
                                                           self.dimensions,
                                                           **variable_parameters
                                                           )
        # Set data values
        output_variable[:] = self.data
        
        # Set variable attributes
        output_variable.setncatts(self.attributes)
        
        return output_variable

{'fraction_non_alphanumeric': 0.07838139238489725, 'fraction_numerical': 0.003823482555360841, 'mean_word_length': 1.5404694455685957, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 79, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3858585', 'n_tokens_mistral': 2661, 'n_tokens_neox': 2558, 'n_words': 868}