<commit_before>###
# Copyright 2011 Diamond Light Source Ltd.
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###

'''
Core package contains wrappers for Java dataset classes
'''

import org.eclipse.january.dataset.Dataset as _ds
import org.eclipse.january.dataset.LazyDataset as _lds
import org.eclipse.january.dataset.DTypeUtils as _dtutils
import org.eclipse.january.dataset.ShapeUtils as _sutils
import org.eclipse.january.dataset.DatasetFactory as _df

import org.eclipse.january.dataset.BooleanDataset as _booleands
import org.eclipse.january.dataset.IntegerDataset as _integerds
import org.eclipse.january.dataset.RGBDataset as _rgbds

import org.eclipse.january.dataset.DatasetUtils as _dsutils
from org.eclipse.scanning.scisoftpy.python.PythonUtils import convertToJava as _cvt2j
from org.eclipse.scanning.scisoftpy.python.PythonUtils import getSlice as _getslice
from org.eclipse.scanning.scisoftpy.python.PythonUtils import setSlice as _setslice
from org.eclipse.scanning.scisoftpy.python.PythonUtils import convertToSlice as _cvt2js
from org.eclipse.scanning.scisoftpy.python.PythonUtils import createFromObject as _create

import org.apache.commons.math3.complex.Complex as _jcomplex #@UnresolvedImport

import Jama.Matrix as _matrix #@UnresolvedImport

import types as _types

import java.lang.ArrayIndexOutOfBoundsException as _jarrayindex_exception #@UnresolvedImport
import java.lang.IllegalArgumentException as _jillegalargument_exception #@UnresolvedImport

class ndgeneric(object):
    pass # there is no array scalars at the moment

generic = ndgeneric

newaxis = None

class _dtype(object):
    '''
    Dataset type has two properties:

    value = Java dataset type object
    elements = number of elements per item
    '''
    def __init__(self, value, elements=1, name=None):
        self.value = value
        self.elements = elements
        self.name = name
    def __str__(self):
        if self.name:
            s = self.name
        else:
            s = ""
        if self.elements > 1:
            s += "(%d)" % self.elements
        return s

bool = _dtype(_ds.BOOL, name='bool') #@ReservedAssignment
int8 = _dtype(_ds.INT8, name='int8')
int16 = _dtype(_ds.INT16, name='int16')
int32 = _dtype(_ds.INT32, name='int32')
int64 = _dtype(_ds.INT64, name='int64')
cint8 = lambda e : _dtype(_ds.ARRAYINT8, e, 'cint8')
cint16 = lambda e : _dtype(_ds.ARRAYINT16, e, 'cint16')
cint32 = lambda e : _dtype(_ds.ARRAYINT32, e, 'cint32')
cint64 = lambda e : _dtype(_ds.ARRAYINT64, e, 'cint64')
float32 = _dtype(_ds.FLOAT32, name='float32')
float64 = _dtype(_ds.FLOAT64, name='float64')
cfloat32 = lambda e : _dtype(_ds.ARRAYFLOAT32, e, 'cfloat32')
cfloat64 = lambda e : _dtype(_ds.ARRAYFLOAT64, e, 'cfloat64')
complex64 = _dtype(_ds.COMPLEX64, name='complex64')
complex128 = _dtype(_ds.COMPLEX128, name='complex128')
string = _dtype(_ds.STRING, name='S')
rgb = _dtype(_ds.RGB, 3, 'rgb')

# tuple of floating point types
_floattype = (_ds.FLOAT32, _ds.FLOAT64, _ds.ARRAYFLOAT32, _ds.ARRAYFLOAT64)

# dictionaries to map from Java dataset types to Jython types
__jdtype2jytype = { _ds.BOOL : bool, _ds.INT8 : int8, _ds.INT16 : int16,
                    _ds.INT32 : int32, _ds.INT64 : int64,
                    _ds.FLOAT32 : float32, _ds.FLOAT64 : float64,
                    _ds.COMPLEX64 : complex64, _ds.COMPLEX128 : complex128,
                    _ds.STRING : string, _ds.RGB : rgb }

__jcdtype2jytype = { _ds.ARRAYINT8 : cint8, _ds.ARRAYINT16 : cint16,
                    _ds.ARRAYINT32 : cint32, _ds.ARRAYINT64 : cint64,
                    _ds.ARRAYFLOAT32 : cfloat32, _ds.ARRAYFLOAT64 : cfloat64 }

# get dtype from object
def _getdtypefromobj(jobj):
    jdtype = _dtutils.getDTypeFromObject(jobj)
    if jdtype in __jdtype2jytype:
        return __jdtype2jytype[jdtype]
    raise ValueError, "Java dataset type unknown"

# get dtype from Java dataset
def _getdtypefromjdataset(jobj):
    d = jobj.getDType()
    if d in __jdtype2jytype:
        return __jdtype2jytype[d]
    if d in __jcdtype2jytype:
        return __jcdtype2jytype[d](jobj.getElementsPerItem())
    raise ValueError, "Java dataset type unknown"

# check for native python type
def _translatenativetype(dtype):
    if dtype is None:
        return None
    elif isinstance(dtype, _dtype) or dtype in __jcdtype2jytype.values():
        return dtype
    elif dtype is int:
        return int32
    elif dtype is float:
        return float64
    elif dtype is complex:
        return complex128
    raise ValueError, "Dataset type is not recognised"

# default types
int_ = int32 # TODO should be 64 for 64-bit OS
float_ = float64
complex_ = complex128

# native types
int = int #@ReservedAssignment
float = float #@ReservedAssignment
complex = complex #@ReservedAssignment

from jarray import array as _array
_arraytype = type(_array([0], 'f')) # this is used for testing if returned object is a Java array

import java.util.List as _jlist #@UnresolvedImport
import java.util.Map as _jmap #@UnresolvedImport


def Sciwrap(a):
    '''
    This wrapper function is required for any Java method that returns a dataset
    '''
    if a is None:
        raise ValueError, "No value given"
    if isinstance(a, _jcomplex): # convert to complex
        return complex(a.getReal(), a.getImaginary())
    if isinstance(a, ndarray):
        return a
    if isinstance(a, _rgbds):
        return ndarrayRGB(buffer=a)
    if isinstance(a, _ds):
        return ndarray(buffer=a)
    return a

def _jinput(arg): # strip for java input
    if type(arg) is _types.DictType:
        d = dict()
        for k,v in arg.items():
            d[_jinput(k)] = _jinput(v)
        return d
    elif type(arg) is _types.ListType:
        return [ _jinput(a) for a in arg ]
    elif type(arg) is _types.TupleType:
        return tuple([ _jinput(a) for a in arg ])
    elif isinstance(arg, _jlist):
        return [ _jinput(a) for a in arg ]
    elif type(arg) is _arraytype:
        return [ _jinput(a) for a in arg if a is not None]
    elif isinstance(arg, ndarray):
        return arg._jdataset()
    elif isinstance(arg, complex):
        return _jcomplex(arg.real, arg.imag)

    return arg

def _joutput(result): # wrap java output
    t = type(result)
    if t is _types.ListType or t is _types.TupleType or isinstance(result, _jlist):
        return tuple([ Sciwrap(r) for r in result ])
    elif t is _arraytype:
        return [ Sciwrap(r) for r in result if r is not None ]
    return Sciwrap(result)

from decorator import decorator as _decorator
<commit_msg>Wrapped import<commit_after>###
# Copyright 2011 Diamond Light Source Ltd.
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###

'''
Core package contains wrappers for Java dataset classes
'''

import org.eclipse.january.dataset.Dataset as _ds
import org.eclipse.january.dataset.LazyDataset as _lds
import org.eclipse.january.dataset.DTypeUtils as _dtutils
import org.eclipse.january.dataset.ShapeUtils as _sutils
import org.eclipse.january.dataset.DatasetFactory as _df

import org.eclipse.january.dataset.BooleanDataset as _booleands
import org.eclipse.january.dataset.IntegerDataset as _integerds
import org.eclipse.january.dataset.RGBDataset as _rgbds

import org.eclipse.january.dataset.DatasetUtils as _dsutils
from org.eclipse.scanning.scisoftpy.python.PythonUtils import convertToJava as _cvt2j
from org.eclipse.scanning.scisoftpy.python.PythonUtils import getSlice as _getslice
from org.eclipse.scanning.scisoftpy.python.PythonUtils import setSlice as _setslice
from org.eclipse.scanning.scisoftpy.python.PythonUtils import convertToSlice as _cvt2js
from org.eclipse.scanning.scisoftpy.python.PythonUtils import createFromObject as _create

import org.apache.commons.math3.complex.Complex as _jcomplex #@UnresolvedImport

import Jama.Matrix as _matrix #@UnresolvedImport

import types as _types

import java.lang.ArrayIndexOutOfBoundsException as _jarrayindex_exception #@UnresolvedImport
import java.lang.IllegalArgumentException as _jillegalargument_exception #@UnresolvedImport

class ndgeneric(object):
    pass # there is no array scalars at the moment

generic = ndgeneric

newaxis = None

class _dtype(object):
    '''
    Dataset type has two properties:

    value = Java dataset type object
    elements = number of elements per item
    '''
    def __init__(self, value, elements=1, name=None):
        self.value = value
        self.elements = elements
        self.name = name
    def __str__(self):
        if self.name:
            s = self.name
        else:
            s = ""
        if self.elements > 1:
            s += "(%d)" % self.elements
        return s

bool = _dtype(_ds.BOOL, name='bool') #@ReservedAssignment
int8 = _dtype(_ds.INT8, name='int8')
int16 = _dtype(_ds.INT16, name='int16')
int32 = _dtype(_ds.INT32, name='int32')
int64 = _dtype(_ds.INT64, name='int64')
cint8 = lambda e : _dtype(_ds.ARRAYINT8, e, 'cint8')
cint16 = lambda e : _dtype(_ds.ARRAYINT16, e, 'cint16')
cint32 = lambda e : _dtype(_ds.ARRAYINT32, e, 'cint32')
cint64 = lambda e : _dtype(_ds.ARRAYINT64, e, 'cint64')
float32 = _dtype(_ds.FLOAT32, name='float32')
float64 = _dtype(_ds.FLOAT64, name='float64')
cfloat32 = lambda e : _dtype(_ds.ARRAYFLOAT32, e, 'cfloat32')
cfloat64 = lambda e : _dtype(_ds.ARRAYFLOAT64, e, 'cfloat64')
complex64 = _dtype(_ds.COMPLEX64, name='complex64')
complex128 = _dtype(_ds.COMPLEX128, name='complex128')
string = _dtype(_ds.STRING, name='S')
rgb = _dtype(_ds.RGB, 3, 'rgb')

# tuple of floating point types
_floattype = (_ds.FLOAT32, _ds.FLOAT64, _ds.ARRAYFLOAT32, _ds.ARRAYFLOAT64)

# dictionaries to map from Java dataset types to Jython types
__jdtype2jytype = { _ds.BOOL : bool, _ds.INT8 : int8, _ds.INT16 : int16,
                    _ds.INT32 : int32, _ds.INT64 : int64,
                    _ds.FLOAT32 : float32, _ds.FLOAT64 : float64,
                    _ds.COMPLEX64 : complex64, _ds.COMPLEX128 : complex128,
                    _ds.STRING : string, _ds.RGB : rgb }

__jcdtype2jytype = { _ds.ARRAYINT8 : cint8, _ds.ARRAYINT16 : cint16,
                    _ds.ARRAYINT32 : cint32, _ds.ARRAYINT64 : cint64,
                    _ds.ARRAYFLOAT32 : cfloat32, _ds.ARRAYFLOAT64 : cfloat64 }

# get dtype from object
def _getdtypefromobj(jobj):
    jdtype = _dtutils.getDTypeFromObject(jobj)
    if jdtype in __jdtype2jytype:
        return __jdtype2jytype[jdtype]
    raise ValueError, "Java dataset type unknown"

# get dtype from Java dataset
def _getdtypefromjdataset(jobj):
    d = jobj.getDType()
    if d in __jdtype2jytype:
        return __jdtype2jytype[d]
    if d in __jcdtype2jytype:
        return __jcdtype2jytype[d](jobj.getElementsPerItem())
    raise ValueError, "Java dataset type unknown"

# check for native python type
def _translatenativetype(dtype):
    if dtype is None:
        return None
    elif isinstance(dtype, _dtype) or dtype in __jcdtype2jytype.values():
        return dtype
    elif dtype is int:
        return int32
    elif dtype is float:
        return float64
    elif dtype is complex:
        return complex128
    raise ValueError, "Dataset type is not recognised"

# default types
int_ = int32 # TODO should be 64 for 64-bit OS
float_ = float64
complex_ = complex128

# native types
int = int #@ReservedAssignment
float = float #@ReservedAssignment
complex = complex #@ReservedAssignment

from jarray import array as _array
_arraytype = type(_array([0], 'f')) # this is used for testing if returned object is a Java array

import java.util.List as _jlist #@UnresolvedImport
import java.util.Map as _jmap #@UnresolvedImport


def Sciwrap(a):
    '''
    This wrapper function is required for any Java method that returns a dataset
    '''
    if a is None:
        raise ValueError, "No value given"
    if isinstance(a, _jcomplex): # convert to complex
        return complex(a.getReal(), a.getImaginary())
    if isinstance(a, ndarray):
        return a
    if isinstance(a, _rgbds):
        return ndarrayRGB(buffer=a)
    if isinstance(a, _ds):
        return ndarray(buffer=a)
    return a

def _jinput(arg): # strip for java input
    if type(arg) is _types.DictType:
        d = dict()
        for k,v in arg.items():
            d[_jinput(k)] = _jinput(v)
        return d
    elif type(arg) is _types.ListType:
        return [ _jinput(a) for a in arg ]
    elif type(arg) is _types.TupleType:
        return tuple([ _jinput(a) for a in arg ])
    elif isinstance(arg, _jlist):
        return [ _jinput(a) for a in arg ]
    elif type(arg) is _arraytype:
        return [ _jinput(a) for a in arg if a is not None]
    elif isinstance(arg, ndarray):
        return arg._jdataset()
    elif isinstance(arg, complex):
        return _jcomplex(arg.real, arg.imag)

    return arg

def _joutput(result): # wrap java output
    t = type(result)
    if t is _types.ListType or t is _types.TupleType or isinstance(result, _jlist):
        return tuple([ Sciwrap(r) for r in result ])
    elif t is _arraytype:
        return [ Sciwrap(r) for r in result if r is not None ]
    return Sciwrap(result)

try:
    from decorator import decorator as _decorator
except Exception, e:
    print >> sys.stderr, "Could not import decorator"
    print >> sys.stderr, e

{'fraction_non_alphanumeric': 0.08969842164599774, 'fraction_numerical': 0.026071025930101466, 'mean_word_length': 3.4506114769520226, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3951832', 'n_tokens_mistral': 4653, 'n_tokens_neox': 4518, 'n_words': 1408}