<commit_before>from netCDF4 import Dataset
import numpy
from datetime import datetime, timedelta

class Ragged():
    '''
    object to represent a ragged array, with some helper functions.
    '''

    def __init__(self, filename):
        '''
        filename: name of netcdf file containing wod profiles
        '''
        self.rootgrp = Dataset(filename, "r", format="NETCDF4")

    def ncasts(self):
        return self.rootgrp.dimensions['casts'].size

    def variables(self):
        return self.rootgrp.variables

class Profile():
    '''
    object to represent a single wodpy-compatible profile object
    '''

    ## generic helpers and format mungers

    def __init__(self, ragged, index):
        '''
        ragged: a Ragged object
        index: index of cast from Ragged object to turn into a profile
        '''

        self.r = ragged
        self.i = index

    def decode_bytearray(self, bytearray):
        '''
        decode a numpy masked array of bytes into a regular string
        '''

        return ''.join([a.decode('UTF-8') for i, a in enumerate(bytearray) if not bytearray.mask[i]])

    def determine_offset(self, var):
        '''
        determine the offset in the list of measurements for <var> where this profile's data begins
        '''

        previous = self.r.variables()[var][0:self.i]
        if not previous.mask is numpy.ma.nomask:
            return sum([int(a) for i, a in enumerate(previous) if not previous.mask[i]])
        else:
            return sum([int(a) for a in previous])

    ## metadata

    def latitude(self):
        return self.r.variables()['lat'][self.i].item()

    def latitude_unc(self):
        return None

    def longitude(self):
        return self.r.variables()['lon'][self.i].item()

    def longitude_unc(self):
        return None

    def uid(self):
        return self.r.variables()['wod_unique_cast'][self.i].item()

    def n_levels(self):
        return self.r.variables()['z_row_size'][self.i].item()

    def _date(self):
        return self.r.variables()['date'][self.i].item()

    def year(self):
        return int(str(self._date())[0:4])

    def month(self):
        return int(str(self._date())[4:6])

    def day(self):
        return int(str(self._date())[6:8])

    def time(self):
        return self.r.variables()['GMT_time'][self.i].item()

    def datetime(self):
        """ Returns the date and time as a datetime object. """

        time  = self.time()
        if time is None or time < 0 or time >= 24:
            time = 0

        try:
            d = datetime(self.year(), self.month(), self.day()) + timedelta(hours=time)
            return d
        except:
            return

    def cruise(self):

        fullcruise = self.decode_bytearray(self.r.variables()['WOD_cruise_identifier'][self.i])
        return int(fullcruise[2:])

    def PIs(self):
        offset = self.determine_offset('Primary_Investigator_rowsize')
        nentries = self.r.variables()['Primary_Investigator_rowsize'][self.i].item()
        pis = self.r.variables()['Primary_Investigator'][offset:offset + nentries]
        return [self.decode_bytearray(a) for a in pis]

    def PIs_var(self):
        offset = self.determine_offset('Primary_Investigator_rowsize')
        nentries = self.r.variables()['Primary_Investigator_rowsize'][self.i].item()
        vars = self.r.variables()['Primary_Investigator_VAR'][offset:offset + nentries]
        return [self.decode_bytearray(a) for a in vars]

    def originator_cruise(self):
        return self.decode_bytearray(self.r.variables()['originators_cruise_identifier'][self.i])

    def originator_station(self):
        return self.r.variables()['Orig_Stat_Num'][self.i].item()

    def extract_secondary_header(self, index):
        return None

    def originator_flag_type(self):
        return None

    def probe_type(self, raw=False):
        # probe type; by default converts back to index from https://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf,
        # for backwards compatibility. set raw=True to get the string directly from the netCDF dataset variable.
        
        probe = self.decode_bytearray(self.r.variables()['dataset'][self.i])
        if raw:
            return probe
        else:
            probecodes = {
                "unknown": 0,
                "MBT": 1,
                "XBT": 2,
                "DBT": 3,
                "CTD": 4,
                "STD": 5,
                "XCTD": 6,
                "bottle/rossette/net": 7,
                "underway/intake": 8,
                "profling float": 9,
                "moored buoy": 10,
                "drifting buoy": 11,
                "towed CTD": 12,
                "animal mounted": 13,
                "bucket": 14,<commit_msg>unpack global file attrs<commit_after>from netCDF4 import Dataset
import numpy
from datetime import datetime, timedelta

class Ragged():
    '''
    object to represent a ragged array, with some helper functions.
    '''

    def __init__(self, filename):
        '''
        filename: name of netcdf file containing wod profiles
        '''
        self.rootgrp = Dataset(filename, "r", format="NETCDF4")

    def ncasts(self):
        return self.rootgrp.dimensions['casts'].size

    def variables(self):
        return self.rootgrp.variables

    def attributes(self):
        return self.rootgrp.ncattrs()

    def get_global_attr(self, attr):
        # unpack a global attribute from this ragged array.
        if attr in self.attributes():
            return self.rootgrp.getncattr(attr)
        else:
            print('Attribute ' + attr + ' not found in this ragged array. Valid options are: ')
            print(self.attributes())
            return None

class Profile():
    '''
    object to represent a single wodpy-compatible profile object
    '''

    ## generic helpers and format mungers

    def __init__(self, ragged, index):
        '''
        ragged: a Ragged object
        index: index of cast from Ragged object to turn into a profile
        '''

        self.r = ragged
        self.i = index

    def decode_bytearray(self, bytearray):
        '''
        decode a numpy masked array of bytes into a regular string
        '''

        return ''.join([a.decode('UTF-8') for i, a in enumerate(bytearray) if not bytearray.mask[i]])

    def determine_offset(self, var):
        '''
        determine the offset in the list of measurements for <var> where this profile's data begins
        '''

        previous = self.r.variables()[var][0:self.i]
        if not previous.mask is numpy.ma.nomask:
            return sum([int(a) for i, a in enumerate(previous) if not previous.mask[i]])
        else:
            return sum([int(a) for a in previous])

    ## metadata

    def latitude(self):
        return self.r.variables()['lat'][self.i].item()

    def latitude_unc(self):
        return None

    def longitude(self):
        return self.r.variables()['lon'][self.i].item()

    def longitude_unc(self):
        return None

    def uid(self):
        return self.r.variables()['wod_unique_cast'][self.i].item()

    def n_levels(self):
        return self.r.variables()['z_row_size'][self.i].item()

    def _date(self):
        return self.r.variables()['date'][self.i].item()

    def year(self):
        return int(str(self._date())[0:4])

    def month(self):
        return int(str(self._date())[4:6])

    def day(self):
        return int(str(self._date())[6:8])

    def time(self):
        return self.r.variables()['GMT_time'][self.i].item()

    def datetime(self):
        """ Returns the date and time as a datetime object. """

        time  = self.time()
        if time is None or time < 0 or time >= 24:
            time = 0

        try:
            d = datetime(self.year(), self.month(), self.day()) + timedelta(hours=time)
            return d
        except:
            return

    def cruise(self):

        fullcruise = self.decode_bytearray(self.r.variables()['WOD_cruise_identifier'][self.i])
        return int(fullcruise[2:])

    def PIs(self):
        offset = self.determine_offset('Primary_Investigator_rowsize')
        nentries = self.r.variables()['Primary_Investigator_rowsize'][self.i].item()
        pis = self.r.variables()['Primary_Investigator'][offset:offset + nentries]
        return [self.decode_bytearray(a) for a in pis]

    def PIs_var(self):
        offset = self.determine_offset('Primary_Investigator_rowsize')
        nentries = self.r.variables()['Primary_Investigator_rowsize'][self.i].item()
        vars = self.r.variables()['Primary_Investigator_VAR'][offset:offset + nentries]
        return [self.decode_bytearray(a) for a in vars]

    def originator_cruise(self):
        return self.decode_bytearray(self.r.variables()['originators_cruise_identifier'][self.i])

    def originator_station(self):
        return self.r.variables()['Orig_Stat_Num'][self.i].item()

    def extract_secondary_header(self, index):
        return None

    def originator_flag_type(self):
        return None

    def probe_type(self, raw=False):
        # probe type; by default converts back to index from https://data.nodc.noaa.gov/woa/WOD/DOC/wodreadme.pdf,
        # for backwards compatibility. Set raw=True to get the string directly from the netCDF dataset variable.

        probe = self.decode_bytearray(self.r.variables()['dataset'][self.i])
        if raw:
            return probe
        else:
            probecodes = {
                "unknown": 0,
                "MBT": 1,
                "XBT": 2,
                "DBT": 3,
                "CTD": 4,
                "STD": 5,
                "XCTD": 6,
                "bottle/rossette/net": 7,
                "underway/intake": 8,
                "profling float": 9,
                "moored buoy": 10,
                "drifting buoy": 11,
                "towed CTD": 12,
                "animal mounted": 13,
                "bucket": 14,
{'fraction_non_alphanumeric': 0.12385735811150175, 'fraction_numerical': 0.007031642390758413, 'mean_word_length': 2.2408854166666665, 'pattern_counts': {'":': 30, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '570829', 'n_tokens_mistral': 3091, 'n_tokens_neox': 2908, 'n_words': 748}