<commit_before># Copyright (c) 2013, <NAME>
# All rights reserved.
# This software is made available under a BSD 3-clause license.
# Please refer to the accompanying LICENSE.TXT file.
"""
sdttrans module - scientific data type translator

Includes functions for translating between various scientific data types, principally those used
within the met/ocean software community (numpy, netcdf, nco, etc).

As the source of the richest set of data types, NumPy acts as the central point for all the
supported translations. By which is meant that a type translation from grammar X to grammar Y
is effected as a translation from X to Numpy, then Numpy to Y, thus:

type X => type N => type Y

As an aide-memoire the names and letter codes of the various NumPy scalar types are given below.

NumPy Type              Kind  Code  Remarks
----------              ----  ----  -------
bool_                   'b'   '?'   compat with python bool
bool8

byte (=int8)            'i'   'b'
short (=int16)          'i'   'h'
intc                    'i'   'i'
int_  (=int32)          'i'   'l'   compat with python int
intp                    'i'   'p'
longlong (int64)        'i'   'q'

ubyte (=uint8)          'u'   'B'
ushort (=uint16)        'u'   'H'
uintc                   'u'   'I'
uint (=uint32)          'u'   'L'   compat with python int
uintp                   'u'   'P'
ulonglong (=uint64)     'u'   'Q'

half (=float16)         'f'   'e'
single                  'f'   'f'   compat with C float
double                  'f'         compat with C double
float_ (=float32)       'f'   'd'   compat with python float
longfloat (=float64)    'f'   'g'

str_                    'S'   'S#'  where # = number of elements
unicode_                'U'   'U#'  where # = number of elements
object                        'O'

Note: complex types (kind='c') and void types (kind='V') are omitted from the above table since
they are not currently handled by this module.
"""
import sys
import numpy as np

# Figure out the size of Python int and float types on this platform
if sys.maxint > 2147483647 :
   platform_int  = 'int64'
   platform_uint = 'uint64'
else :
   platform_int  = 'int32'
   platform_uint = 'uint32'

if sys.float_info.mant_dig > 24 :
   platform_float = 'float64'
else :
   platform_float = 'float32'

# Symbolic constants for supported type grammars/ontologies.
(
CDL3_NS,
CDL4_NS,
NCML_NS,
NCO_NS,
NETCDF3_NS,
NETCDF4_NS,
NUMPY_NS
) = range(7)

# Map all non-numpy type names to numpy type names (np.dtype.name)
type_maps = {
   CDL3_NS: {'label': 'CDL3',
      'byte': 'int8', 'short': 'int16', 'int': 'int32', 'long': 'int32',
      'float': 'float32', 'real': 'float32', 'double': 'float64',
      'char': '|S1',
   },
   CDL4_NS: {'label': 'CDL4',
      'byte': 'int8', 'short': 'int16', 'int': 'int32', 'long': 'int32', 'int64': 'int64',
      'ubyte': 'uint8', 'ushort': 'uint16', 'uint': 'uint32', 'uint64': 'uint64',
      'float': 'float32', 'real': 'float32', 'double': 'float64',
      'char': '|S1', 'string': 'string_',
   },
   NCML_NS: {'label': 'NCML',
      'byte': 'int8', 'short': 'int16', 'int': 'int32', 'long': 'int32',
      'float': 'float32', 'double': 'float64',
      'char': '|S1', 'string': 'string_', 'String': 'string_',
   },
   NCO_NS: {'label': 'NCO',
      'b': 'int8', 's': 'int16', 'i': 'int32', 'l': 'int32', 'll': 'int64', 'int64': 'int64',
      'ub': 'uint8', 'us': 'uint16', 'u': 'uint32', 'ui': 'uint32', 'ul': 'int32',
      'ull': 'uint64', 'uint64': 'uint64',
      'f': 'float32', 'd': 'float64',
      'c': '|S1', 'sng': 'string_',
   },
   NETCDF3_NS: {'label': 'NETCDF-3',
      'NC_BYTE': 'int8', 'NC_SHORT': 'int16', 'NC_INT': 'int32', 'NC_LONG': 'int32',
      'NC_FLOAT': 'float32', 'NC_DOUBLE': 'float64', 'NC_CHAR': '|S1',<commit_msg>Fixed typo in NCO typemap in sdttrans module.<commit_after># Copyright (c) 2013, <NAME>
# All rights reserved.
# This software is made available under a BSD 3-clause license.
# Please refer to the accompanying LICENSE.TXT file.
"""
sdttrans module - scientific data type translator

Includes functions for translating between various scientific data types, principally those used
within the met/ocean software community (numpy, netcdf, nco, etc).

As the source of the richest set of data types, NumPy acts as the central point for all the
supported translations. By which is meant that a type translation from grammar X to grammar Y
is effected as a translation from X to Numpy, then Numpy to Y, thus:

type X => type N => type Y

As an aide-memoire the names and letter codes of the various NumPy scalar types are given below.

NumPy Type              Kind  Code  Remarks
----------              ----  ----  -------
bool_                   'b'   '?'   compat with python bool
bool8

byte (=int8)            'i'   'b'
short (=int16)          'i'   'h'
intc                    'i'   'i'
int_  (=int32)          'i'   'l'   compat with python int
intp                    'i'   'p'
longlong (int64)        'i'   'q'

ubyte (=uint8)          'u'   'B'
ushort (=uint16)        'u'   'H'
uintc                   'u'   'I'
uint (=uint32)          'u'   'L'   compat with python int
uintp                   'u'   'P'
ulonglong (=uint64)     'u'   'Q'

half (=float16)         'f'   'e'
single                  'f'   'f'   compat with C float
double                  'f'         compat with C double
float_ (=float32)       'f'   'd'   compat with python float
longfloat (=float64)    'f'   'g'

str_                    'S'   'S#'  where # = number of elements
unicode_                'U'   'U#'  where # = number of elements
object                        'O'

Note: complex types (kind='c') and void types (kind='V') are omitted from the above table since
they are not currently handled by this module.
"""
import sys
import numpy as np

# Figure out the size of Python int and float types on this platform
if sys.maxint > 2147483647 :
   platform_int  = 'int64'
   platform_uint = 'uint64'
else :
   platform_int  = 'int32'
   platform_uint = 'uint32'

if sys.float_info.mant_dig > 24 :
   platform_float = 'float64'
else :
   platform_float = 'float32'

# Symbolic constants for supported type grammars/ontologies.
(
CDL3_NS,
CDL4_NS,
NCML_NS,
NCO_NS,
NETCDF3_NS,
NETCDF4_NS,
NUMPY_NS
) = range(7)

# Map all non-numpy type names to numpy type names (np.dtype.name)
type_maps = {
   CDL3_NS: {'label': 'CDL3',
      'byte': 'int8', 'short': 'int16', 'int': 'int32', 'long': 'int32',
      'float': 'float32', 'real': 'float32', 'double': 'float64',
      'char': '|S1',
   },
   CDL4_NS: {'label': 'CDL4',
      'byte': 'int8', 'short': 'int16', 'int': 'int32', 'long': 'int32', 'int64': 'int64',
      'ubyte': 'uint8', 'ushort': 'uint16', 'uint': 'uint32', 'uint64': 'uint64',
      'float': 'float32', 'real': 'float32', 'double': 'float64',
      'char': '|S1', 'string': 'string_',
   },
   NCML_NS: {'label': 'NCML',
      'byte': 'int8', 'short': 'int16', 'int': 'int32', 'long': 'int32',
      'float': 'float32', 'double': 'float64',
      'char': '|S1', 'string': 'string_', 'String': 'string_',
   },
   NCO_NS: {'label': 'NCO',
      'b': 'int8', 's': 'int16', 'i': 'int32', 'l': 'int32', 'll': 'int64', 'int64': 'int64',
      'ub': 'uint8', 'us': 'uint16', 'u': 'uint32', 'ui': 'uint32', 'ul': 'uint32',
      'ull': 'uint64', 'uint64': 'uint64',
      'f': 'float32', 'd': 'float64',
      'c': '|S1', 'sng': 'string_',
   },
   NETCDF3_NS: {'label': 'NETCDF-3',
      'NC_BYTE': 'int8', 'NC_SHORT': 'int16', 'NC_INT': 'int32', 'NC_LONG': 'int32',
      'NC_FLOAT': 'float32', 'NC_DOUBLE': 'float64', 'NC_CHAR': '|S1',
{'fraction_non_alphanumeric': 0.16620752984389348, 'fraction_numerical': 0.04171585989767808, 'mean_word_length': 2.621852731591449, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6556719', 'n_tokens_mistral': 2886, 'n_tokens_neox': 2673, 'n_words': 896}