<commit_before>#!/usr/bin/env python


'''
@package prototype.science_object_codec
@file prototype/science_object_codec.py
@author <NAME>
@author <NAME>
@brief prototype codec for the science data object

To run the encoder and decoder, please ensure that numpy and h5py are installed. Also make sure that the temporary
folder, /tmp/, is available for use. Any hdf files that need to be temporarily written to the /tmp/ folder will
be automatically cleaned at the end of the execution.

Please follow these sequence of steps to run the demo example.

1. In the pyon directory (where the pyon code base is located), run $bin/pycc
2. In the pycc interactive shell that opens up, run the following command:
    %loadpy prototype/hdf/example/encoder_decoder_demo.py
3. The output will be a numpy array of ones and with shape (4,5).

What is happening above is that the demo script, prototype/hdf/example/encoder_decoder_demo.py, is being run. If you read
the script, you will know the commands to be followed to encode a numpy array (which results in a binary string being
returned) and decode a binary string (which results in a numpy array to be read out). The demo script is essentially
an encode-decode operation.

Alternatively, if you dont want to use the script, run bin/pycc to get into the pycc interactive shell,
and...
1. Create an encoder object: $encoder = HDFEncoder()
2. Add data as an array:  $encoder.add_hdf_dataset('/myGroup/measurements/pressure', numpy.ones((10,200)))
3. Convert all the data to a binary string for easy transportation: $hdf_string = encoder.encoder_close()

4. Create a decoder object: $decoder = HDFDecoder(hdf_string)
5. Read the array out of the decoder: $decoder.read_hdf_dataset('/myGroup/measurements/pressure')

'''


import uuid
import hashlib
import os
import os.path
import sys
from pyon.util.log import log
from pyon.core.exception import IonException
from pyon.util.file_sys import FS_DIRECTORY, FS, FileSystem

class ScienceObjectTransportException(IonException):
    """
    Exception class for science object transport library
    """


class ScienceObjectTransport(object):

    def __init__(self, science_object=None):

        self._science_object = science_object

        self._metadata_packets = {}

        self._data_packets = {}


    def encode(self):
        """
        Produce a packet based on the state of the data object<commit_msg>Luke: hdf_codec pruned imports<commit_after>#!/usr/bin/env python


'''
@package prototype.science_object_codec
@file prototype/science_object_codec.py
@author <NAME>
@author <NAME>
@brief prototype codec for the science data object

To run the encoder and decoder, please ensure that numpy and h5py are installed. Also make sure that the temporary
folder, /tmp/, is available for use. Any hdf files that need to be temporarily written to the /tmp/ folder will
be automatically cleaned at the end of the execution.

Please follow these sequence of steps to run the demo example.

1. In the pyon directory (where the pyon code base is located), run $bin/pycc
2. In the pycc interactive shell that opens up, run the following command:
    %loadpy prototype/hdf/example/encoder_decoder_demo.py
3. The output will be a numpy array of ones and with shape (4,5).

What is happening above is that the demo script, prototype/hdf/example/encoder_decoder_demo.py, is being run. If you read
the script, you will know the commands to be followed to encode a numpy array (which results in a binary string being
returned) and decode a binary string (which results in a numpy array to be read out). The demo script is essentially
an encode-decode operation.

Alternatively, if you dont want to use the script, run bin/pycc to get into the pycc interactive shell,
and...
1. Create an encoder object: $encoder = HDFEncoder()
2. Add data as an array:  $encoder.add_hdf_dataset('/myGroup/measurements/pressure', numpy.ones((10,200)))
3. Convert all the data to a binary string for easy transportation: $hdf_string = encoder.encoder_close()

4. Create a decoder object: $decoder = HDFDecoder(hdf_string)
5. Read the array out of the decoder: $decoder.read_hdf_dataset('/myGroup/measurements/pressure')

'''


import uuid
import hashlib
import os
from pyon.util.log import log
from pyon.core.exception import IonException
from pyon.util.file_sys import FS, FileSystem

class ScienceObjectTransportException(IonException):
    """
    Exception class for science object transport library
    """


class ScienceObjectTransport(object):

    def __init__(self, science_object=None):

        self._science_object = science_object

        self._metadata_packets = {}

        self._data_packets = {}


    def encode(self):
        """
        Produce a packet based on the state of the data object
{'fraction_non_alphanumeric': 0.07018646553530275, 'fraction_numerical': 0.006704378797402053, 'mean_word_length': 4.211790393013101, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1045235', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1283, 'n_words': 601}