<commit_before>"""
Set of programs and tools to read the outputs from RH, 1.5D version
"""
import os
import numpy as np


class Rh15dout:
    def __init__(self, fdir='.', verbose=True):
        self.files = []
        self.params = {}
        self.verbose = verbose
        self.fdir = fdir
        OUTFILE_FUNC = {"output_aux": self.read_aux,
                        "output_indata": self.read_indata,
                        "output_ray": self.read_ray,
                        "output_spectrum": self.read_spectrum}
        for outfile, func in OUTFILE_FUNC.items():
            if (os.path.isfile("%s/%s.ncdf" % (self.fdir, outfile)) or
                os.path.isfile("%s/%s.hdf5" % (self.fdir, outfile))):
                func()

    def read_aux(self, infile=None):
        ''' Reads Aux file. '''
        if infile is None:
            infile = '%s/output_aux.hdf5' % self.fdir
            if not os.path.isfile(infile):  # See if netCDF file exists
                infile = os.path.splitext(infile)[0] + '.ncdf'
        if not os.path.isfile(infile):
            if self.verbose:
                print('File %s not found, skipping.' % infile)
            return
        self.files.append(read_hdf5(self, infile))
        if self.verbose:
            print(('--- Read %s file.' % infile))

    def read_indata(self, infile=None):
        ''' Reads indata file. '''
        if infile is None:
            infile = '%s/output_indata.hdf5' % self.fdir
            if not os.path.isfile(infile):  # See if netCDF file exists
                infile = os.path.splitext(infile)[0] + '.ncdf'
        if not os.path.isfile(infile):
            if self.verbose:
                print('File %s not found, skipping.' % infile)
        self.files.append(read_hdf5(self, infile))
        if self.verbose:
            print(('--- Read %s file.' % infile))

    def read_spectrum(self, infile=None):
        ''' Reads spectrum file. '''
        if infile is None:
            infile = '%s/output_spectrum.ncdf' % self.fdir
        self.spectrum = DataHolder()
        self.files.append(read_hdf5(self.spectrum, infile))
        if self.verbose:
            print(('--- Read %s file.' % infile))

    def read_ray(self, infile=None):
        ''' Reads ray file. '''
        if infile is None:
            infile = '%s/output_ray.hdf5' % self.fdir
            if not os.path.isfile(infile):  # See if netCDF file exists
                infile = os.path.splitext(infile)[0] + '.ncdf'
        if not os.path.isfile(infile):
            if self.verbose:
                print('File %s not found, skipping.' % infile)
        self.ray = DataHolder()
        self.files.append(read_hdf5(self.ray, infile))
        if self.verbose:
            print(('--- Read %s file.' % infile))

    def read_J(self, infile='scratch/J.dat.hdf5'):
        ''' Reads angle averaged intensity file '''
        self.files.append(read_hdf5(self, infile))
        if self.verbose:
            print(('--- Read %s file.' % infile))

    def close(self):
        ''' Closes the open NetCDF files '''
        for f in self.files:
            f.close()


class HDF5Atmos:
    def __init__(self, infile):
        self.file = read_hdf5(self, infile)
        self.closed = False
<commit_msg>Get RH 1.5D new class from xarray_dask branch<commit_after>"""
Set of programs and tools to read the outputs from RH, 1.5D version
"""
import os
import numpy as np
import xarray as xr
import h5py


class Rh15dout:
    def __init__(self, fdir='.', verbose=True):
        self.files = []
        self.params = {}
        self.verbose = verbose
        self.fdir = fdir
        for outfile in ["output_aux", "output_indata"]:
            OUTFILE = os.path.join(self.fdir, "%s.hdf5" % (outfile))
            self.read_groups(OUTFILE)
        RAYFILE = os.path.join(self.fdir, "output_ray.hdf5")
        self.read_ray(RAYFILE)

    def read_groups(self, infile):
        ''' Reads indata file, group by group. '''
        if not os.path.isfile(infile):   # See if netCDF file exists
            infile = os.path.splitext(infile)[0] + '.ncdf'
        if not os.path.isfile(infile):
            if self.verbose:
                print('File %s not found, skipping.' % infile)
        f = h5py.File(infile, "r")
        GROUPS = [g for g in f.keys() if type(f[g]) == h5py._hl.group.Group]
        f.close()
        for g in GROUPS:
            setattr(self, g, xr.open_dataset(infile, group=g))
            self.files.append(getattr(self, g))
        if self.verbose:
            print(('--- Read %s file.' % infile))

    def read_ray(self, infile=None):
        ''' Reads ray file. '''
        if infile is None:
            infile = '%s/output_ray.hdf5' % self.fdir
            if not os.path.isfile(infile):  # See if netCDF file exists
                infile = os.path.splitext(infile)[0] + '.ncdf'
        if not os.path.isfile(infile):
            if self.verbose:
                print('File %s not found, skipping.' % infile)
        self.ray = xr.open_dataset(infile)
        self.files.append(self.ray)
        if self.verbose:
            print(('--- Read %s file.' % infile))

    def close(self):
        ''' Closes the open files '''
        for f in self.files:
            f.close()


class HDF5Atmos:
    def __init__(self, infile):
        self.file = read_hdf5(self, infile)
        self.closed = False

{'fraction_non_alphanumeric': 0.13222678105184632, 'fraction_numerical': 0.006527415143603133, 'mean_word_length': 1.886437029063509, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2534230', 'n_tokens_mistral': 1794, 'n_tokens_neox': 1739, 'n_words': 439}