<commit_before>import sys
import re
import glob
import numpy as np

debug = False

################################################################################
# metafile parsing

# for python2.5
try: next
except NameError:
    def next ( obj ): return obj.next()

_currentline = ''

class ParseError(ValueError):
    def __str__(self):
        metafile = self.args[0]
        lines = self.args[1:]
        try:
            name = metafile.name
        except AttributeError:
            name = metafile

        return '\n'.join(('in metafile: '+name,)
                         + lines
                         + ('in: ' + _currentline,))


# these deal with comments in the metafile

_comment_pattern = re.compile(
        r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
        re.DOTALL | re.MULTILINE
    )

def _comment_replacer(match):
    s = match.group(0)
    if s.startswith('/'):
        return ""
    else:
        return s

def strip_comments(text):
    """ strips C and C++ style comments from text """
    return re.sub(_comment_pattern, _comment_replacer, text)


_string_pattern = re.compile(r"'(.*)'$")

def parse1(s): 
    """ convert one item to appropriate type """
    m = _string_pattern.match(s)
    if m:
        s = m.group(1)
        # unquote quotes
        s = re.sub(r"''","'",s)
        return s

    if '.' in s or 'e' in s.lower():
        return float(s)
    else:
        try:
            return int(s)
        except ValueError:
            raise ParseError("Cannot parse value: " + s)


_closing = {'[':']',
            '{':'}',
           }

def parsemeta(metafile):
    """ parses metafile (file object or filename) into a dictionary of lists
        of floats, ints or strings
    """
    global _currentline

    try:
        lines = open(metafile)
    except TypeError:
        lines = iter(metafile)

    d = {}
    for line in lines:
        line = strip_comments(line)
        # skip empty lines
        if re.match(r'\s*$', line):
            continue

        m = re.match(r' *(\w*) *= *(.*?) *$', line)
        if m:
            key,line = m.groups()
        else:
            raise ParseError(metafile,line)

        # look for the opening delimiter ('[' or '{')
        opening = line[0]
        try:
            closing = _closing[opening]
        except KeyError:
            raise ParseError(metafile,line,'Values must be enclosed in [] or {}.')

        # read more lines until a matching closing delimiter is found
        while closing not in line:
            try:
                nextline = next(lines)
            except StopIteration:
                raise ParseError(metafile,line,'No closing ' + closing + ' found.')

            line += ' ' + strip_comments(nextline).rstrip()

        if line[-2:] != closing + ';':
            raise ParseError(metafile,line,
                             'Values must be enclosed in "[ ];" or "{ };".')

        # remove delimiters
        line = line[1:-2].strip(" ,")
        _currentline = line

        if opening == '[':
            # [] can contain any type of values, separated by commas
            val = [ parse1(s) for s in re.split(r',? *',line) ]
        else:
            # {} can only contain single quote-delimited strings separated by space
            val = [ s.rstrip() for s in re.split(r"'  *'", line.strip("'")) ]

        d[key] = val

    return d

################################################################################

def message(*args):
    sys.stderr.write(' '.join([str(s) for s in args]) + '\n')


def aslist(i):
    """ if iterable, turn into list, otherwise put into list """
    try:
        res = list(i)
    except TypeError:
        res = [i]
    return res


def fromfileshape(filename,dtype,shape=None,**kwargs):
    return np.fromfile(filename, dtype, **kwargs).reshape(shape)


def scanforfiles(fname):
    """ return list of iteration numbers for which metafiles with base fname exist """
    import glob
    allfiles = glob.glob(fname + '.' + 10*'[0-9]' + '.001.001.meta')
    if len(allfiles) == 0:
        allfiles = glob.glob(fname + '.' + 10*'[0-9]' + '.meta')
        off = -5
    else:
        off = -13

    itrs = [ int(s[off-10:off]) for s in allfiles ]
    itrs.sort()
    return itrs


def readmeta(f):
    """ read meta file and extract tile/timestep-specific parameters """
    meta = parsemeta(f)
    dimList = meta.pop('dimList')
    # pythonize
    gdims = tuple(dimList[-3::-3])
    i0s   = [ i-1 for i in dimList[-2::-3] ]
    ies   = dimList[-1::-3]
    # remove file-specific parameters
    timeInterval = meta.pop('timeInterval', None)
    timeStepNumber = meta.pop('timeStepNumber', None)
    map2gl = meta.pop('map2glob', None)
    # put back only global dimensions
    meta['dimList'] = list(gdims[::-1])
    return gdims,i0s,ies,timeStepNumber,timeInterval,map2gl,meta


_typeprefixes = {'ieee-be':'>',
                 'b'      :'>',
                 '>'      :'>',
                 'ieee-le':'<',
                 'l'      :'<',
                 '<'      :'<',
                }
_typesuffixes = {'float32':'f4',
                 'float64':'f8',
                }

def rdmds(fnamearg,itrs=-1,machineformat='b',rec=None,fill_value=0,
          returnmeta=False,astype=float,region=None,lev=(),
          usememmap=False,mm=False,squeeze=True):
    """ a     = rdmds(fname,...)
    a     = rdmds(fname,itrs,...)
    a,its,meta = rdmds(fname,...,returnmeta=True)

    Read meta-data files as written by MITgcm.

    Without itrs, will try to read

      fname.meta or fname.001.001.meta, ...

    If itrs is a list of integers of an integer, it will read the corresponding

      fname.000000iter.meta, ...

    If itrs is NaN, it will read all iterations for which files are found.
    If itrs is Inf, it will read the highest iteration found.

    Returns:

        a    :: numpy array of the data read
        its  :: list of iteration numbers read (only if itrs=NaN or Inf)
        meta :: dictionary of metadata (only if returnmeta=True)

    Keyword arguments:

        machineformat :: endianness ('b' or 'l', default 'b')
        rec           :: list of records to read (default all)
        fill_value    :: fill value for missing (blank) tiles (default 0)
        astype        :: data type to return (default: double precision)
                         None: keep data type/precision of file
        region        :: (x0,x1,y0,y1) read only this region (default (0,nx,0,ny))
        lev           :: list of levels to read, or, for multiple dimensions
                         (excluding x,y), tuple(!) of lists
        usememmap     :: if True, use a memory map for reading data (default False)
                         recommended when using lev, or region with global files
                         to save memory and, possibly, time

    Examples:

        XC = rdmds('XC')
        XC = rdmds('res_*/XC')
        T = rdmds('T.0000002880')
        T = rdmds('T',2880)
        T = rdmds('T',[2880,5760])
        T,its = rdmds('T',numpy.Inf)
        VVEL = rdmds('pickup',2880,rec=range(50,100))
        a = rdmds('diags',2880,rec=0,lev=[5])
        a = rdmds('diags',2880,rec=0,lev=([0],[0,1,5,6,7]))
        from numpy import r_
        a = rdmds('diags',2880,rec=0,lev=([0],r_[:2,5:8]))
    """
    usememmap = usememmap or mm
    if usememmap:
        readdata = np.memmap
    else:
        readdata = fromfileshape

    # add iteration number to file name unless itrs is -1
    additrs = itrs != -1
    if itrs is np.nan:
        # all iterations
        itrs = scanforfiles(fnamearg)
        message('Reading {0} time levels: '.format(len(itrs)), *itrs)
        returnits = True
        itrsislist = True<commit_msg>augment doc string<commit_after>import sys
import re
import glob
import numpy as np

debug = False

################################################################################
# metafile parsing

# for python2.5
try: next
except NameError:
    def next ( obj ): return obj.next()

_currentline = ''

class ParseError(ValueError):
    def __str__(self):
        metafile = self.args[0]
        lines = self.args[1:]
        try:
            name = metafile.name
        except AttributeError:
            name = metafile

        return '\n'.join(('in metafile: '+name,)
                         + lines
                         + ('in: ' + _currentline,))


# these deal with comments in the metafile

_comment_pattern = re.compile(
        r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
        re.DOTALL | re.MULTILINE
    )

def _comment_replacer(match):
    s = match.group(0)
    if s.startswith('/'):
        return ""
    else:
        return s

def strip_comments(text):
    """ strips C and C++ style comments from text """
    return re.sub(_comment_pattern, _comment_replacer, text)


_string_pattern = re.compile(r"'(.*)'$")

def parse1(s): 
    """ convert one item to appropriate type """
    m = _string_pattern.match(s)
    if m:
        s = m.group(1)
        # unquote quotes
        s = re.sub(r"''","'",s)
        return s

    if '.' in s or 'e' in s.lower():
        return float(s)
    else:
        try:
            return int(s)
        except ValueError:
            raise ParseError("Cannot parse value: " + s)


_closing = {'[':']',
            '{':'}',
           }

def parsemeta(metafile):
    """ parses metafile (file object or filename) into a dictionary of lists
        of floats, ints or strings
    """
    global _currentline

    try:
        lines = open(metafile)
    except TypeError:
        lines = iter(metafile)

    d = {}
    for line in lines:
        line = strip_comments(line)
        # skip empty lines
        if re.match(r'\s*$', line):
            continue

        m = re.match(r' *(\w*) *= *(.*?) *$', line)
        if m:
            key,line = m.groups()
        else:
            raise ParseError(metafile,line)

        # look for the opening delimiter ('[' or '{')
        opening = line[0]
        try:
            closing = _closing[opening]
        except KeyError:
            raise ParseError(metafile,line,'Values must be enclosed in [] or {}.')

        # read more lines until a matching closing delimiter is found
        while closing not in line:
            try:
                nextline = next(lines)
            except StopIteration:
                raise ParseError(metafile,line,'No closing ' + closing + ' found.')

            line += ' ' + strip_comments(nextline).rstrip()

        if line[-2:] != closing + ';':
            raise ParseError(metafile,line,
                             'Values must be enclosed in "[ ];" or "{ };".')

        # remove delimiters
        line = line[1:-2].strip(" ,")
        _currentline = line

        if opening == '[':
            # [] can contain any type of values, separated by commas
            val = [ parse1(s) for s in re.split(r',? *',line) ]
        else:
            # {} can only contain single quote-delimited strings separated by space
            val = [ s.rstrip() for s in re.split(r"'  *'", line.strip("'")) ]

        d[key] = val

    return d

################################################################################

def message(*args):
    sys.stderr.write(' '.join([str(s) for s in args]) + '\n')


def aslist(i):
    """ if iterable, turn into list, otherwise put into list """
    try:
        res = list(i)
    except TypeError:
        res = [i]
    return res


def fromfileshape(filename,dtype,shape=None,**kwargs):
    return np.fromfile(filename, dtype, **kwargs).reshape(shape)


def scanforfiles(fname):
    """ return list of iteration numbers for which metafiles with base fname exist """
    import glob
    allfiles = glob.glob(fname + '.' + 10*'[0-9]' + '.001.001.meta')
    if len(allfiles) == 0:
        allfiles = glob.glob(fname + '.' + 10*'[0-9]' + '.meta')
        off = -5
    else:
        off = -13

    itrs = [ int(s[off-10:off]) for s in allfiles ]
    itrs.sort()
    return itrs


def readmeta(f):
    """ read meta file and extract tile/timestep-specific parameters """
    meta = parsemeta(f)
    dimList = meta.pop('dimList')
    # pythonize
    gdims = tuple(dimList[-3::-3])
    i0s   = [ i-1 for i in dimList[-2::-3] ]
    ies   = dimList[-1::-3]
    # remove file-specific parameters
    timeInterval = meta.pop('timeInterval', None)
    timeStepNumber = meta.pop('timeStepNumber', None)
    map2gl = meta.pop('map2glob', None)
    # put back only global dimensions
    meta['dimList'] = list(gdims[::-1])
    return gdims,i0s,ies,timeStepNumber,timeInterval,map2gl,meta


_typeprefixes = {'ieee-be':'>',
                 'b'      :'>',
                 '>'      :'>',
                 'ieee-le':'<',
                 'l'      :'<',
                 '<'      :'<',
                }
_typesuffixes = {'float32':'f4',
                 'float64':'f8',
                }

def rdmds(fnamearg,itrs=-1,machineformat='b',rec=None,fill_value=0,
          returnmeta=False,astype=float,region=None,lev=(),
          usememmap=False,mm=False,squeeze=True):
    """ a     = rdmds(fname,...)
    a     = rdmds(fname,itrs,...)
    a,its,meta = rdmds(fname,...,returnmeta=True)

    Read meta-data files as written by MITgcm.

    Without itrs, will try to read

      fname.meta or fname.001.001.meta, ...

    If itrs is a list of integers of an integer, it will read the corresponding

      fname.000000iter.meta, ...

    If itrs is NaN, it will read all iterations for which files are found.
    If itrs is Inf, it will read the highest iteration found.

    fname may contain shell wildcards, which is useful for tile files organized
    into directories, e.g.,

      T = rdmds('prefix*/T', 2880)

    will read prefix0000/T.0000002880.*, prefix0001/T.0000002880.*, ...
    (and any others that match the wildcard, so be careful how you name things!)

    Returns:

        a    :: numpy array of the data read
        its  :: list of iteration numbers read (only if itrs=NaN or Inf)
        meta :: dictionary of metadata (only if returnmeta=True)

    Keyword arguments:

        machineformat :: endianness ('b' or 'l', default 'b')
        rec           :: list of records to read (default all)
                         useful for pickups and multi-field diagnostics files
        fill_value    :: fill value for missing (blank) tiles (default 0)
        astype        :: data type to return (default: double precision)
                         None: keep data type/precision of file
        region        :: (x0,x1,y0,y1) read only this region (default (0,nx,0,ny))
        lev           :: list of levels to read, or, for multiple dimensions
                         (excluding x,y), tuple(!) of lists (see examples below)
        usememmap     :: if True, use a memory map for reading data (default False)
                         recommended when using lev, or region with global files
                         to save memory and, possibly, time

    Examples:

        XC = rdmds('XC')
        XC = rdmds('res_*/XC')
        T = rdmds('T.0000002880')
        T = rdmds('T',2880)
        T2 = rdmds('T',[2880,5760])
        T,its = rdmds('T',numpy.Inf)
        VVEL = rdmds('pickup',2880,rec=range(50,100))
        a5 = rdmds('diags',2880,rec=0,lev=[5])
        a = rdmds('diags',2880,rec=0,lev=([0],[0,1,5,6,7]))
        from numpy import r_
        a = rdmds('diags',2880,rec=0,lev=([0],r_[:2,5:8]))  # same as previous
        a = rdmds('diags',2880,rec=0)[0, [0,1,5,6,7], ...]  # same, but less efficient
        a = rdmds('diags',2880)[0, 0, [0,1,5,6,7], ...]     # even less efficient
    """
    usememmap = usememmap or mm
    if usememmap:
        readdata = np.memmap
    else:
        readdata = fromfileshape

    # add iteration number to file name unless itrs is -1
    additrs = itrs != -1
    if itrs is np.nan:
        # all iterations
        itrs = scanforfiles(fnamearg)
        message('Reading {0} time levels: '.format(len(itrs)), *itrs)
        returnits = True
        itrsislist = True
{'fraction_non_alphanumeric': 0.15532286212914484, 'fraction_numerical': 0.01994515083520319, 'mean_word_length': 2.050380228136882, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5618698', 'n_tokens_mistral': 5257, 'n_tokens_neox': 4963, 'n_words': 1386}