<commit_before>"""
Functions for reading and writing Bruker binary (ser/fid) files, Bruker
JCAMP-DX parameter (acqus) files, and Bruker pulse program (pulseprogram)
files.
"""

from __future__ import print_function, division

__developer_info__ = """
Bruker file format information
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Bruker binary files (ser/fid) store data as an array of numbers whose
endianness is determined by the parameter BYTORDA (1 = big endian, 0 = little
endian), and whose data type is determined by the parameter DTYPA (0 = int32,
2 = float64). Typically the direct dimension is digitally filtered. The exact
method of removing this filter is unknown but an approximation is available.

Bruker JCAMP-DX files (acqus, etc) are text file which are described by the
`JCAMP-DX standard <http://www.jcamp-dx.org/>`_.  Bruker parameters are
prefixed with a '$'.

Bruker pulseprogram files are text files described in various Bruker manuals.
Of special important are lines which describe external variable assignments
(surrounded by "'s), loops (begin with lo), phases (contain ip of dp) or
increments (contain id, dd, ipu or dpu).  These lines are parsed when reading
the file with nmrglue.

"""

from functools import reduce
import os
from warnings import warn

import numpy as np

from . import fileiobase
from ..process import proc_base


# data creation

def create_data(data):
    """
    Create a bruker data array (recast into a complex128 or int32)
    """
    if np.iscomplexobj(data):
        return np.array(data, dtype='complex128')
    else:
        return np.array(data, dtype='int32')


# universal dictionary functions

def guess_udic(dic, data, strip_fake=False):
    """
    Guess parameters of universal dictionary from dic, data pair.

    Parameters
    ----------
    dic : dict
        Dictionary of Bruker parameters.
    data : ndarray
        Array of NMR data.
    strip_fake: bool
        If data is proceed (i.e. read using `bruker.read_pdata`) and the Bruker
        processing parameters STSI and/or STSR are set, the returned sweep
        width and carrier frequencies is changed to values that are incorrect
        but instead can are intended to trick the normal unit_conversion object
        into producing the correct result.

    Returns
    -------
    udic : dict
        Universal dictionary of spectral parameters.

    """
    # TODO if pprog, acqus, procs are in dic use them better.

    # create an empty universal dictionary
    udic = fileiobase.create_blank_udic(data.ndim)

    # update default values
    for b_dim in range(data.ndim):
        udic[b_dim]["size"] = data.shape[b_dim]

        # try to add additional parameter from acqus dictionary keys
        try:
            add_axis_to_udic(udic, dic, b_dim, strip_fake)
        except:
            warn("Failed to determine udic parameters for dim: %i" % (b_dim))
    return udic


def add_axis_to_udic(udic, dic, udim, strip_fake):
    """
    Add axis parameters to a udic.

    Parameters
    ----------
    udic : dict
        Universal dictionary to update, modified in place.
    dic : dict
        Bruker dictionary used to determine axes parameters.
    dim : int
        Universal dictionary dimension to update.
    strip_fake: bool
        See `bruker.guess_udic`

    """
    # This could still use some work
    b_dim = udic['ndim'] - udim - 1  # last dim
    acq_file = "acqu" + str(b_dim + 1) + "s"
    pro_file = "proc" + str(b_dim + 1) + "s"

    # Because they're inconsistent,..
    if acq_file == "acqu1s":
        acq_file = "acqus"

    if pro_file == "proc1s":
        pro_file = "procs"

    if acq_file in dic:
        sw = dic[acq_file]["SW_h"]
    elif pro_file in dic:
        sw = dic[pro_file]["SW_p"]
        # procNs files store sw (in Hz) with the 'SW_p' key instead of 'SW_h'.
        # this is a bug in TopSpin (TopSpin3.5pl7)

    if acq_file in dic:
        udic[udim]["label"] = dic[acq_file]["NUC1"]
    elif pro_file in dic:
        udic[udim]["label"] = dic[pro_file]["AXNUC"]

    try:
        obs = dic[pro_file]["SF"]
        if acq_file in dic:
            car = (dic[acq_file]["SFO1"] - obs) * 1e6
        else:
            # we should be able to use the 'OFFSET' parameter in procNs to
            # calculate 'car'. But this is slightly off (~ 5E-3 Hz)
            # most likely because the procs file does not store the OFFSET
            # to a high precision. Hence the value in acquNs is given priority
            car = dic[pro_file]["OFFSET"]*obs - sw/2

    except KeyError:
        warn('The chemical shift referencing was not corrected for "sr".')
        obs = dic[acq_file]["SFO1"]
        car = dic[acq_file]["O1"]

    if strip_fake:
        try:
            # Temporary parameters
            w = sw/float(dic[pro_file]["FTSIZE"])
            d = (w * dic[pro_file]["STSR"]) + (w * dic[pro_file]["STSI"]/2.0)

            # Fake car frequency
            car -= (d-(sw/2.0))

            # Fake sw frequency
            sw = w * dic[pro_file]["STSI"]
        except KeyError:
            pass

    udic[udim]["sw"] = sw
    udic[udim]["car"] = car
    udic[udim]["obs"] = obs

    if acq_file in dic:
        if acq_file == "acqus":
            if dic['acqus']['AQ_mod'] == 0:     # qf
                udic[udim]['complex'] = False
            else:
                udic[udim]['complex'] = True
        else:
            aq_mod = dic[acq_file]["FnMODE"]
            if aq_mod == 0:
                udic[udim]["encoding"] = "undefined"
            elif aq_mod == 1:
                udic[udim]["encoding"] = "magnitude"  # qf
            elif aq_mod == 2:
                udic[udim]["encoding"] = "magnitude"  # qsec
            elif aq_mod == 3:
                udic[udim]["encoding"] = "tppi"
            elif aq_mod == 4:
                udic[udim]["encoding"] = "states"
            elif aq_mod == 5:
                udic[udim]["encoding"] = "states-tppi"  # states-tppi
            elif aq_mod == 6:
                udic[udim]["encoding"] = "echo-antiecho"  # echo-antiecho
    else:
        if pro_file == "procs":
            # this seems to have the 'MC2' parameter always set to 0
            # irrespective of what the actual data is
            udic[udim]["complex"] = "undefined"
        else:
            # these are only used when params in acquNs are 'undefined'
            # but in absence of acqus, this is the best that can be done
            aq_mod = dic[pro_file]["MC2"]
            if aq_mod == 0:
                udic[udim]["encoding"] = "magnitude"  # qf
            elif aq_mod == 1:
                udic[udim]["encoding"] = "magnitude"  # qsec
            elif aq_mod == 2:
                udic[udim]["encoding"] = "tppi"
            elif aq_mod == 3:
                udic[udim]["encoding"] = "states"
            elif aq_mod == 4:
                udic[udim]["encoding"] = "states-tppi"
            elif aq_mod == 5:
                udic[udim]["encoding"] = "echo-antiecho"

    return udic


def create_dic(udic):
    """
    Create a Bruker parameter dictionary from a universal dictionary.

    Parameters
    ----------
    udic : dict
        Universal dictionary of spectral parameters.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.

    """
    ndim = udic['ndim']

    # determine the size in bytes
    if udic[ndim - 1]["complex"]:
        bytes = 8
    else:
        bytes = 4

    for k in range(ndim):
        bytes *= udic[k]["size"]

    dic = {"FILE_SIZE": bytes}

    # create the pprog dictionary parameter
    dic["pprog"] = {'incr': [[], [1]] * (ndim * 2 - 2),
                    'loop': [2] * (ndim * 2 - 2),
                    'ph_extra': [[]] * (ndim * 2 - 2),
                    'phase': [[]] * (ndim * 2 - 2),
                    'var': {}}

    # create acqus dictionary parameters and fill in loop sizes
    dic['acqus'] = create_acqus_dic(udic[ndim - 1], direct=True)
    if ndim >= 2:
        dic["acqu2s"] = create_acqus_dic(udic[ndim - 2])
        dic["pprog"]["loop"][1] = udic[ndim - 2]["size"] // 2
    if ndim >= 3:
        dic["acqu3s"] = create_acqus_dic(udic[ndim - 3])
        dic["pprog"]["loop"][3] = udic[ndim - 3]["size"] // 2
    if ndim >= 4:
        dic["acqu4s"] = create_acqus_dic(udic[ndim - 4])
        dic["pprog"]["loop"][5] = udic[ndim - 4]["size"] // 2

    return dic


def create_acqus_dic(adic, direct=False):
    """
    Create a Bruker acqus dictionary from an Universal axis dictionary.
    Set direct=True for direct dimension.
    """
    if adic["complex"]:
        AQ_mod = 3
        if direct:
            TD = int(np.ceil(adic["size"] / 256.) * 256) * 2
        else:
            TD = adic["size"]
    else:
        AQ_mod = 1
        if direct:
            TD = int(np.ceil(adic["size"] / 256.) * 256)
        else:
            TD = adic["size"]

    s = '##NMRGLUE automatically created parameter file'
    return {'_comments': [], '_coreheader': [s], 'AQ_mod': AQ_mod, 'TD': TD}


# Global read/write function and related utilities

def read(dir=".", bin_file=None, acqus_files=None, pprog_file=None, shape=None,
         cplex=None, big=None, isfloat=None, read_pulseprogram=True,
         read_acqus=True, procs_files=None, read_procs=True):
    """
    Read Bruker files from a directory.

    Parameters
    ----------
    dir : str
        Directory to read from.
    bin_file : str, optional
        Filename of binary file in directory. None uses standard files.
    acqus_files : list, optional
        List of filename(s) of acqus parameter files in directory. None uses
        standard files.
    pprog_file : str, optional
        Filename of pulse program in directory. None uses standard files.
    shape : tuple, optional
        Shape of resulting data.  None will guess the shape from the spectral
        parameters.
    cplex : bool, optional
        True is direct dimension is complex, False otherwise. None will guess
        quadrature from spectral parameters.
    big : bool or None, optional
        Endianness of binary file. True for big-endian, False for
        little-endian, None to determine endianness from acqus file(s).
    isfloat : bool or None, optional
        Data type of binary file. True for float64, False for int32. None to
        determine data type from acqus file(s).
    read_pulseprogram : bool, optional
        True to read pulse program, False prevents reading.
    read_acqus : bool, optional
        True to read acqus files(s), False prevents reading.
    procs_files : list, optional
        List of filename(s) of procs parameter files in directory. None uses
        standard files.
    read_procs : bool, optional
        True to read procs files(s), False prevents reading.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    data : ndarray
        Array of NMR data.

    See Also
    --------
    read_pdata : Read Bruker processed files.
    read_lowmem : Low memory reading of Bruker files.
    write : Write Bruker files.

    """
    if os.path.isdir(dir) is not True:
        raise IOError("directory %s does not exist" % (dir))

    # Take a shot at reading the procs file
    if read_procs:
        dic = read_procs_file(dir, procs_files)
    else:
        # create an empty dictionary
        dic = dict()

    # determine parameter automatically
    if bin_file is None:
        if os.path.isfile(os.path.join(dir, "fid")):
            bin_file = "fid"
        elif os.path.isfile(os.path.join(dir, "ser")):
            bin_file = "ser"

        # Look two directory levels lower.
        elif os.path.isdir(os.path.dirname(os.path.dirname(dir))):

            # ! change the dir
            dir = os.path.dirname(os.path.dirname(dir))

            if os.path.isfile(os.path.join(dir, "fid")):
                bin_file = "fid"
            elif os.path.isfile(os.path.join(dir, "ser")):
                bin_file = "ser"
            else:
                mesg = "No Bruker binary file could be found in %s"
                raise IOError(mesg % (dir))
        else:
            mesg = "No Bruker binary file could be found in %s"
            raise IOError(mesg % (dir))

    if read_acqus:
        # read the acqus_files and add to the dictionary
        acqus_dic = read_acqus_file(dir, acqus_files)
        dic = _merge_dict(dic, acqus_dic)

    if pprog_file is None:
        pprog_file = "pulseprogram"

    # read the pulse program and add to the dictionary
    if read_pulseprogram:
        try:
            dic["pprog"] = read_pprog(os.path.join(dir, pprog_file))
        except:
            warn('Error reading the pulse program')

    # determine file size and add to the dictionary
    dic["FILE_SIZE"] = os.stat(os.path.join(dir, bin_file)).st_size

    # determine shape and complexity for direct dim if needed
    if shape is None or cplex is None:
        gshape, gcplex = guess_shape(dic)
        if gcplex is True:    # divide last dim by 2 if complex
            t = list(gshape)
            t[-1] = t[-1] // 2
            gshape = tuple(t)
    if shape is None:
        shape = gshape
    if cplex is None:
        cplex = gcplex

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "acqus" in dic and "BYTORDA" in dic["acqus"]:
            if dic["acqus"]["BYTORDA"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "acqus" in dic and "DTYPA" in dic["acqus"]:
            if dic["acqus"]["DTYPA"] == 2:
                isfloat = True
            else:
                isfloat = False

    # read the binary file
    f = os.path.join(dir, bin_file)
    null, data = read_binary(f, shape=shape, cplex=cplex, big=big,
                             isfloat=isfloat)
    return dic, data


def read_lowmem(dir=".", bin_file=None, acqus_files=None, pprog_file=None,
                shape=None, cplex=None, big=None, isfloat=None,
                read_pulseprogram=True, read_acqus=True, procs_files=None,
                read_procs=True):
    """
    Read Bruker files from a directory using minimal amounts of memory.

    See :py:func:`read` for Parameters.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    data : array_like
        Low memory object which can access NMR data on demand.

    See Also
    --------
    read : Read Bruker files.
    write_lowmem : Write Bruker files using minimal amounts of memory.

    """

    if os.path.isdir(dir) is not True:
        raise IOError("directory %s does not exist" % (dir))

    # Take a shot at reading the procs file
    if read_procs:
        dic = read_procs_file(dir, procs_files)
    else:
        # create an empty dictionary
        dic = dict()

    # determine parameter automatically
    if bin_file is None:
        if os.path.isfile(os.path.join(dir, "fid")):
            bin_file = "fid"
        elif os.path.isfile(os.path.join(dir, "ser")):
            bin_file = "ser"

        # Look two directory levels lower.
        elif os.path.isdir(os.path.dirname(os.path.dirname(dir))):

            # ! change the dir
            dir = os.path.dirname(os.path.dirname(dir))

            if os.path.isfile(os.path.join(dir, "fid")):
                bin_file = "fid"
            elif os.path.isfile(os.path.join(dir, "ser")):
                bin_file = "ser"
            else:
                mesg = "No Bruker binary file could be found in %s"
                raise IOError(mesg % (dir))
        else:
            mesg = "No Bruker binary file could be found in %s"
            raise IOError(mesg % (dir))

    if read_acqus:
        # read the acqus_files and add to the dictionary
        acqus_dic = read_acqus_file(dir, acqus_files)
        dic = _merge_dict(dic, acqus_dic)

    if pprog_file is None:
        pprog_file = "pulseprogram"

    # read the pulse program and add to the dictionary
    if read_pulseprogram:
        dic["pprog"] = read_pprog(os.path.join(dir, pprog_file))

    # determine file size and add to the dictionary
    dic["FILE_SIZE"] = os.stat(os.path.join(dir, bin_file)).st_size

    # determine shape and complexity for direct dim if needed
    if shape is None or cplex is None:
        gshape, gcplex = guess_shape(dic)
        if gcplex is True:    # divide last dim by 2 if complex
            t = list(gshape)
            t[-1] = t[-1] // 2
            gshape = tuple(t)
    if shape is None:
        shape = gshape
    if cplex is None:
        cplex = gcplex

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "acqus" in dic and "BYTORDA" in dic["acqus"]:
            if dic["acqus"]["BYTORDA"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "acqus" in dic and "DTYPA" in dic["acqus"]:
            if dic["acqus"]["DTYPA"] == 2:
                isfloat = True
            else:
                isfloat = False

    # read the binary file
    f = os.path.join(dir, bin_file)
    null, data = read_binary_lowmem(f, shape=shape, cplex=cplex, big=big,
                                    isfloat=isfloat)
    return dic, data


def read_acqus_file(dir='.', acqus_files=None):
    """
    Read Bruker acquisition files from a directory.

    Parameters
    ----------
    dir : str
        Directory to read from.
    acqus_files : list, optional
        List of filename(s) of acqus parameter files in directory. None uses
        standard files.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    """
    if acqus_files is None:
        acqus_files = []
        for f in ["acqus", "acqu2s", "acqu3s", "acqu4s"]:
            if os.path.isfile(os.path.join(dir, f)):
                acqus_files.append(f)

    # create an empty dictionary
    dic = dict()

    # read the acqus_files and add to the dictionary
    for f in acqus_files:
        dic[f] = read_jcamp(os.path.join(dir, f))
    return dic


def read_procs_file(dir='.', procs_files=None):
    """
    Read Bruker processing files from a directory.

    Parameters
    ----------
    dir : str
        Directory to read from.
    procs_files : list, optional
        List of filename(s) of procs parameter files in directory. None uses
        standard files.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    """
    if procs_files is None:
        procs_files = []
        for f in ["procs", "proc2s", "proc3s", "proc4s"]:
            if os.path.isfile(os.path.join(dir, f)):
                procs_files.append(f)
        pdata_path = dir

    if procs_files == []:
        if os.path.isdir(os.path.join(dir, 'pdata')):
            pdata_folders = [folder for folder in 
                             os.walk(os.path.join(dir, 'pdata'))][0][1]
            if '1' in pdata_folders:
                pdata_path = os.path.join(dir, 'pdata', '1')
            else:
                pdata_path = os.path.join(dir, 'pdata', pdata_folders[0]) 
                
    for f in ["procs", "proc2s", "proc3s", "proc4s"]:
            if os.path.isfile(os.path.join(pdata_path, f)):
                procs_files.append(f)

    # create an empty dictionary
    dic = dict()

    # read the acqus_files and add to the dictionary
    for f in procs_files:
        dic[f] = read_jcamp(os.path.join(pdata_path, f))
    return dic


def write(dir, dic, data, bin_file=None, acqus_files=None, procs_files=None,
          pprog_file=None, overwrite=False, big=None, isfloat=None, 
          write_prog=True, write_acqus=True, write_procs=False, 
          pdata_folder=False):
    """
    Write Bruker files to disk.

    Parameters
    ----------
    dir : str
        Directory to write files to.
    dir : dict
        Dictionary of Bruker parameters.
    data : array_like
        Array of NMR data
    bin_file : str, optional
        Filename of binary file in directory. None uses standard files.
    acqus_files : list, optional
        List of filename(s) of acqus parameter files in directory. None uses
        standard files.
    procs_file : list, optional
        List of filename(s) of procs parameter files (to write out). None uses a
        list of standard files
    pprog_file : str, optional
        Filename of pulse program in directory. None uses standard files.
    overwrite : bool, optional
        Set True to overwrite files, False will raise a Warning if files
        exist.
    big : bool or None, optional
        Endianness of binary file. True for big-endian, False for
        little-endian, None to determine endianness from Bruker dictionary.
    isfloat : bool or None, optional
        Data type of binary file. True for float64, False for int32. None to
        determine data type from Bruker dictionary.
    write_pprog : bool, optional
        True to write the pulse program file, False prevents writing.
    write_acqus : bool, optional
        True to write the acqus files(s), False prevents writing.
    write_procs : bool, optional
        True to write the procs files(s), False prevents writing.
    pdata_folder : int, optional
        Makes a folder and a subfolder ('pdata/pdata_folder') inside the given
        directory where pdata_folder is an integer. procN and procNs files are 
        stored inside pdata_folder. pdata_folder=False (or =0) does not make the 
        pdata folder and pdata_folder=True makes folder '1'.

    See Also
    --------
    write_lowmem : Write Bruker files using minimal amounts of memory.
    read : Read Bruker files.

    """
    # determine parameters automatically
    if bin_file is None:
        if data.ndim == 1:
            bin_file = "fid"
        else:
            bin_file = "ser"

    if acqus_files is None:
        acq = ["acqus", "acqu2s", "acqu3s", "acqu4s"]
        acqus_files = [k for k in acq if (k in dic)]
        acqu_files = [k[:-1] for k in acqus_files]

    if procs_files is None:
        proc = ["procs", "proc2s", "proc3s", "proc4s"]
        procs_files = [k for k in proc if (k in dic)]
        proc_files = [k[:-1] for k in procs_files]

    if pprog_file is None:
        pprog_file = "pulseprogram"

    # write out the acqus files
    if write_acqus:
        for f in acqus_files:
            write_jcamp(dic[f], os.path.join(dir, f), overwrite=overwrite)
        for f in acqu_files:
            write_jcamp(dic[f+'s'], os.path.join(dir, f), overwrite=overwrite)

    # write out the procs files
    if write_procs:
        if pdata_folder is not False:
            try:
                procno = str(int(pdata_folder))
                pdata_path = os.path.join(dir, 'pdata', procno)
            except ValueError:
                raise ValueError('pdata_folder should be an integer')
            
            if not os.path.isdir(pdata_path):
                os.makedirs(pdata_path)
        else:
            pdata_path = dir
    
        for f in procs_files:
            write_jcamp(dic[f], os.path.join(pdata_path, f))
        for f in proc_files:
            write_jcamp(dic[f+'s'], os.path.join(pdata_path, f))

    # write out the pulse program
    if write_prog:
        write_pprog(os.path.join(dir, pprog_file), dic["pprog"],
                    overwrite=overwrite)

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "acqus" in dic and "BYTORDA" in dic["acqus"]:
            if dic["acqus"]["BYTORDA"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "acqus" in dic and "DTYPA" in dic["acqus"]:
            if dic["acqus"]["DTYPA"] == 2:
                isfloat = True
            else:
                isfloat = False

    # write out the binary data
    bin_full = os.path.join(dir, bin_file)
    write_binary(bin_full, dic, data, big=big, isfloat=isfloat,
                 overwrite=overwrite)
    return


def write_lowmem(dir, dic, data, bin_file=None, acqus_files=None,
                 pprog_file=None, overwrite=False, big=None, isfloat=None,
                 write_prog=True, write_acqus=True):
    """
    Write Bruker files using minimal amounts of memory (trace by trace).

    See :py:func:`write` for Parameters.

    See Also
    --------
    write : Write Bruker files.
    read_lowmem : Read Bruker files using minimal amounts of memory.

    """
    # determine parameters automatically
    if bin_file is None:
        if data.ndim == 1:
            bin_file = "fid"
        else:
            bin_file = "ser"

    if acqus_files is None:
        acq = ["acqus", "acqu2s", "acqu3s", "acqu4s"]
        acqus_files = [k for k in acq if (k in dic)]

    if pprog_file is None:
        pprog_file = "pulseprogram"

    # write out the acqus files
    if write_acqus:
        for f in acqus_files:
            write_jcamp(dic[f], os.path.join(dir, f), overwrite=overwrite)

    # write out the pulse program
    if write_prog:
        write_pprog(os.path.join(dir, pprog_file), dic["pprog"],
                    overwrite=overwrite)

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "acqus" in dic and "BYTORDA" in dic["acqus"]:
            if dic["acqus"]["BYTORDA"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "acqus" in dic and "DTYPA" in dic["acqus"]:
            if dic["acqus"]["DTYPA"] == 2:
                isfloat = True
            else:
                isfloat = False

    # write out the binary data
    bin_full = os.path.join(dir, bin_file)
    write_binary_lowmem(bin_full, dic, data, big=big, isfloat=isfloat,
                        overwrite=overwrite)
    return


def write_pdata(dir, dic, data, shape=None, submatrix_shape=None, 
        scale_data=True, bin_file=None, procs_files=None, 
        write_procs=False, pdata_folder=False, overwrite=False, 
        big=None, isfloat=None, restrict_access=True):
    """
    Write processed Bruker files to disk.

    Parameters
    ----------
    dir : str
        Directory to write files to.
    dic : dict
        Dictionary of Bruker parameters.
    data : array_like
        Array of NMR data
    shape : tuple, optional
        Shape of data, if file is to be written with a shape
        different than data.shape
    submatrix_shape : tuple, optional
        Shape of the submatrix used to store data (using Bruker specifications)
        If this is not given, the submatrix shape will be guessed from dic
    scale_data : Bool
        True to apply reverse scaling using the scaling guessed from dic
    bin_file : str, optional
        Filename of binary file in directory. None uses standard files.
    procs_file : list, optional
        List of filename(s) of procs parameter files (to write out). None uses a
        list of standard files
    write_procs : Bool
        True to write out the procs files
    pdata_folder : int, optional
        Makes a folder and a subfolder ('pdata/pdata_folder') inside the given
        directory where pdata_folder is an integer. All files (procs and data) are 
        stored inside pdata_folder. pdata_folder=False (or =0) does not make the 
        pdata folder and pdata_folder=True makes folder '1'.
    overwrite : bool, optional
        Set True to overwrite files, False will raise a Warning if files
        exist.
    big : bool or None, optional
        Endianness of binary file. True for big-endian, False for
        little-endian, None to determine endianness from Bruker dictionary.
    isfloat : bool or None, optional
        Data type of binary file. True for float64, False for int32. None to
        determine data type from Bruker dictionary.
    restrict_access : not implemented

    """
   
    # see that data consists of only real elements
    data = data.real

    if scale_data:
        data = scale_pdata(dic, data, reverse=True)

    # see if the dimensionality is given
    # else, set it to the dimensions of data
    if shape == None:
        shape = data.shape

    # guess data dimensionality
    ndim = len(shape)

    # update PARMODE in dictionary
    # This is required when writing back 1D slices from a 2D, 2D planes of 3D, etc
    dic['procs']['PPARMOD'] = ndim - 1

    # reorder the submatrix according 
    if submatrix_shape is None:
        submatrix_shape = guess_shape_and_submatrix_shape(dic)[1]

    data = reorder_submatrix(data, shape, submatrix_shape, reverse=True)

    # see if pdata_folder needs to make and set write path
    if pdata_folder is not False:
        try:
            procno = str(int(pdata_folder))
            pdata_path = os.path.join(dir, 'pdata', procno)
        except ValueError:
            raise ValueError('pdata_folder should be an integer')

        if not os.path.isdir(pdata_path):
            os.makedirs(pdata_path)
    else:
        pdata_path = dir

    # write out the procs files only for the desired dimensions
    if write_procs:
        if procs_files == None:
            proc = ['procs'] + ['proc{}s'.format(i) for i in range(2, ndim+1)]
            procs_files = [f for f in proc if (f in dic)]
    
        for f in procs_files:
            write_jcamp(dic[f], os.path.join(pdata_path, f),
                    overwrite=overwrite)
            write_jcamp(dic[f], os.path.join(pdata_path, f[:-1]),
                    overwrite=overwrite)

    if bin_file == None:
        bin_file = str(ndim) + 'r'*ndim

    bin_full = os.path.join(pdata_path, bin_file)
    write_binary(bin_full, dic, data, big=big, isfloat=isfloat,
                 overwrite=overwrite)
    return
        

def guess_shape(dic):
    """
    Determine data shape and complexity from Bruker dictionary.

    Returns
    -------
    shape : tuple
        Shape of data in Bruker binary file (R+I for all dimensions).
    cplex : bool
        True for complex data in last (direct) dimension, False otherwise.

    """
    # determine complexity of last (direct) dimension
    try:
        aq_mod = dic["acqus"]["AQ_mod"]
    except KeyError:
        aq_mod = 0

    if aq_mod == 0 or aq_mod == 2:
        cplex = False
    elif aq_mod == 1 or aq_mod == 3:
        cplex = True
    else:
        raise ValueError("Unknown Aquisition Mode")

    # file size
    try:
        fsize = dic["FILE_SIZE"]
    except KeyError:
        warn("cannot determine shape do to missing FILE_SIZE key")
        return (1,), True

    # extract td0,td1,td2,td3 from dictionaries
    try:
        td0 = float(dic["acqus"]["TD"])
    except KeyError:
        td0 = 1024   # default value

    try:
        td2 = int(dic["acqu2s"]["TD"])
    except KeyError:
        td2 = 0     # default value

    try:
        td1 = float(dic["acqu3s"]["TD"])
    except KeyError:
        td1 = int(td2)   # default value

    try:
        td3 = int(dic["acqu4s"]["TD"])
    except KeyError:
        td3 = int(td1)     # default value

    # From the acquisition reference manual (section on parameter NBL):
    #     ---
    #     If TD is not a multiple of 256 (1024 bytes), successive FIDs will
    #     still begin at 1024 byte memory boundaries. This is so for the FIDs
    #     in the acquisition memory as well as on disk. The size of the raw
    #     data file (ser) is therefore always a multiple of 1024 times NBL.
    #     ---
    # This seems to hold for 1D data sets as well. However, this paragraph
    # assumes that each data point is 4 bytes, hence the "multiple of 256".
    # For data in DTYPA=2 (float64), each point is 8 bytes, so while it always
    # allocates the fids in 1024-byte blocks, for float64 data it pads the data
    # (by points) out to multiples of 128, not 256. So we need to get the
    # data type before we guess the shape of the last dimension.

    # extract data type from dictionary
    try:
        dtypa = int(dic["acqus"]["DTYPA"])
    except KeyError:
        dtypa = 0   # default value, int32 data

    # last (direct) dimension is given by "TD" parameter in acqus file
    # rounded up to nearest (1024/(bytes per point))
    # next-to-last dimension may be given by "TD" in acqu2s. In 3D+ data
    # this is often the sum of the indirect dimensions
    if dtypa == 2:
        shape = [0, 0, td2, int(np.ceil(td0 / 128.) * 128.)]
    else:
        shape = [0, 0, td2, int(np.ceil(td0 / 256.) * 256.)]

    # additional dimension given by data size
    if shape[2] != 0 and shape[3] != 0:
        shape[1] = fsize // (shape[3] * shape[2] * 4)
        shape[0] = fsize // (shape[3] * shape[2] * shape[1] * 4)

    # if there in no pulse program parameters in dictionary return currect
    # shape after removing zeros
    if "pprog" not in dic or "loop" not in dic["pprog"]:
        return tuple([int(i) for i in shape if i > 1]), cplex

    # if pulseprogram dictionary is missing loop or incr return current shape
    pprog = dic["pprog"]
    if "loop" not in pprog or "incr" not in pprog:
        return tuple([int(i) for i in shape if i > 1]), cplex

    # determine indirect dimension sizes from pulseprogram parameters
    loop = pprog["loop"]
    loopn = len(loop)       # number of loops
    li = [len(i) for i in pprog["incr"]]    # length of incr lists

    # replace td0,td1,td2,td3 in loop list
    rep = {'td0': td0, 'td1': td1, 'td2': td2, 'td3': td3}
    for i, v in enumerate(loop):
        if v in rep.keys():
            loop[i] = rep[v]

    # if the loop variables contains strings, return current shape
    # these variables could be resolved from the var key in the pprog dict
    # but this would require executing unknown code to perform the
    # arithmetic present in the string.
    if str in [type(e) for e in loop]:
        return tuple([int(i) for i in shape if i > 1]), cplex

    # size of indirect dimensions based on number of loops in pulse program
    # there are two kinds of loops, active and passive.
    # active loops are from indirect dimension increments, the corresponding
    # incr lists should have non-zero length and the size of the dimension
    # is twice that of the active loop size.
    # passive loops are from phase cycles and similar elements, these should
    # have zero length incr lists and should be of length 2.

    # The following checks for these and updates the indirect dimension
    # if the above is found.
    if loopn == 1:    # 2D with no leading passive loops
        if li[0] != 0:
            shape[2] = loop[0]
            shape = shape[-2:]

    elif loopn == 2:  # 2D with one leading passive loop
        if loop[0] == 2 and li[0] == 0 and li[1] != 0:
            shape[2] = 2 * loop[1]
            shape = shape[-2:]

    elif loopn == 3:  # 2D with two leading passive loops
        if (loop[0] == 2 and loop[1] == 2 and li[0] == 0 and li[1] == 0
                and li[2] != 0):
            shape[2] = 2 * loop[2]
            shape = shape[-2:]

    elif loopn == 4:  # 3D with one leading passive loop for each indirect dim
        if loop[0] == 2 and li[0] == 0 and li[1] != 0:
            shape[2] = 2 * loop[1]
        if loop[2] == 2 and li[2] == 0 and li[3] != 0:
            shape[1] = 2 * loop[3]
            shape = shape[-3:]

    elif loopn == 5:  # 3D with two/one leading passive loops
        if loop[1] == 2 and li[0] == 0 and li[1] == 0 and li[2] != 0:
            shape[2] = 2 * loop[2]
        if loop[3] == 2 and li[0] == 0 and li[3] == 0 and li[4] != 0:
            shape[1] = 2 * loop[4]
            shape = shape[-3:]

    elif loopn == 6:  # 4D with one leading passive loop for each indirect dim
        if loop[0] == 2 and li[0] == 0 and li[1] != 0:
            shape[2] = 2 * loop[1]
        if loop[2] == 2 and li[2] == 0 and li[3] != 0:
            shape[1] = 2 * loop[3]
        if loop[4] == 2 and li[4] == 0 and li[5] != 0:
            shape[0] = 2 * loop[5]

    elif loopn == 7:
        if loop[1] == 2 and li[0] == 0 and li[1] == 0 and li[2] != 0:
            shape[2] = 2 * loop[2]
        if loop[3] == 2 and li[0] == 0 and li[3] == 0 and li[4] != 0:
            shape[1] = 2 * loop[4]
        if loop[5] == 2 and li[0] == 0 and li[5] == 0 and li[6] != 0:
            shape[0] = 2 * loop[6]

    return tuple([int(i) for i in shape if i >= 2]), cplex


# Bruker processed binary (1r, 1i, 2rr, 2ri, etc) reading

def read_pdata(dir=".", bin_files=None, procs_files=None, read_procs=True,
               acqus_files=None, read_acqus=True, scale_data=True, shape=None,
               submatrix_shape=None, all_components=False, big=None,
               isfloat=None):
    """
    Read processed Bruker files from a directory.

    In Topspin and other programs this data is typically scaled by dividing by
    2 ** -NC_proc where NC_proc is defined in the procs file.  This scaling
    can be accomplished by setting the scale_data parameter to True.

    Parameters
    ----------
    dir : str
        Directory to read from.
    bin_files : list of str, optional
        List of filename of binary file in directory. None uses standard
        files.
    procs_files : list, optional
        List of filename(s) of procs parameter files in directory. None uses
        standard files.
    read_procs : bool, optional
        True to read procs files(s), False prevents reading.
    acqus_files : list, optional
        List of filename(s) of acqus parameter files in directory. None uses
        standard files.
    read_acqus : bool, optional
        True to read acqus files(s), False prevents reading.
    scale_data : bool, optional
        True, the default, to apply scaling defined in the procs file.  The
        data should almost always be scaled. False, returns the
        data as it appears in the file.
    shape : tuple, optional
        Shape of resulting data.  None will guess the shape from the
        parameters in the procs file(s).
    submatrix_shape : tuple, optional
        Shape of submatrix for 2D+ data.  None will guess the shape from
        the metadata in the procs file(s).
    all_components : bool
        True to return a list of all components, False returns just the
        all real component (1r, 2rr, 3rrr, etc).
    big : bool or None, optional
        Endianness of binary file. True for big-endian, False for
        little-endian, None to determine endianness from procs file(s).
    isfloat : bool or None, optional
        Data type of binary file. True for float64, False for int32. None to
        determine data type from procs file(s).

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    data : ndarray or list
        Array of NMR data.  If all_components is True this is a list of array
        with each quadrature component.

    Notes
    -----
    There is currently no support for writing Bruker processed files or
    reading processed files using minimal memory.

    """
    # TODO read_pdata_lowmem, write_pdata

    if os.path.isdir(dir) is not True:
        raise IOError("directory %s does not exist" % (dir))

    # find binary files
    if bin_files is None:
        if os.path.isfile(os.path.join(dir, "1r")):
            if all_components:
                bin_files = ['1r', '1i']
            else:
                bin_files = ['1r']
        elif os.path.isfile(os.path.join(dir, "2rr")):
            if all_components:
                bin_files = ['2rr', '2ri', '2ir', '2ii']
            else:
                bin_files = ['2rr']
        elif os.path.isfile(os.path.join(dir, "3rrr")):
            if all_components:
                bin_files = ['3rrr', '3rri', '3rir', '3rii', '3irr', '3iri',
                             '3iir', '3iii']
            else:
                bin_files = ['3rrr']
        else:
            raise IOError("No Bruker binary file could be found in %s" % (dir))

    if read_procs:
        # read the procs_files and add to the dictionary
        dic = read_procs_file(dir, procs_files)
    else:
        # create an empty dictionary
        dic = dict()

    if read_acqus:
        # If acqus files were not listed check in the usual place.
        acqus_dir = os.path.dirname(os.path.dirname(dir))
        if acqus_files is not None:
            acqus_dic = read_acqus_file(dir, acqus_files)
            # Merge the two dicts.
            dic = _merge_dict(dic, acqus_dic)

        elif os.path.isdir(acqus_dir):
            acqus_dic = read_acqus_file(acqus_dir)
            # Merge the two dicts.
            dic = _merge_dict(dic, acqus_dic)

    # determine shape and complexity for direct dim if needed
    if submatrix_shape is None or shape is None:
        g_shape, g_submatrix_shape = guess_shape_and_submatrix_shape(dic)
        if shape is None:
            shape = g_shape
        if submatrix_shape is None:
            submatrix_shape = g_submatrix_shape

    # issue a warning is submatrix_shape or shape are still None
    if submatrix_shape is None:
        warn('Submatrix shape not defined, returning 1D data')
    if shape is None:
        warn('Data shape not defined, returning 1D data')

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "procs" in dic and "BYTORDP" in dic["procs"]:
            if dic["procs"]["BYTORDP"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "procs" in dic and "DTYPP" in dic["procs"]:
            if dic["procs"]["DTYPP"] == 2:
                isfloat = True
            else:
                isfloat = False

    # read the binary file
    data = [read_pdata_binary(os.path.join(dir, f),
                              shape, submatrix_shape, big, isfloat)[1]
            for f in bin_files]

    # scale data if requested
    if scale_data:
        data = [scale_pdata(dic, d) for d in data]

    if len(data) == 1:
        return dic, data[0]
    else:
        return dic, data


def scale_pdata(dic, data, reverse=False):
    """
    Scale Bruker processed data using parameters from the procs file.

    Parameters
    ----------
    dic : dict
        Dictionary of Bruker parameters.<commit_msg>removed reverse scaling option in scale_pdata, this will be handled by array_to_int function, removed scale_pdata options from the write_pdata function<commit_after>"""
Functions for reading and writing Bruker binary (ser/fid) files, Bruker
JCAMP-DX parameter (acqus) files, and Bruker pulse program (pulseprogram)
files.
"""

from __future__ import print_function, division

__developer_info__ = """
Bruker file format information
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Bruker binary files (ser/fid) store data as an array of numbers whose
endianness is determined by the parameter BYTORDA (1 = big endian, 0 = little
endian), and whose data type is determined by the parameter DTYPA (0 = int32,
2 = float64). Typically the direct dimension is digitally filtered. The exact
method of removing this filter is unknown but an approximation is available.

Bruker JCAMP-DX files (acqus, etc) are text file which are described by the
`JCAMP-DX standard <http://www.jcamp-dx.org/>`_.  Bruker parameters are
prefixed with a '$'.

Bruker pulseprogram files are text files described in various Bruker manuals.
Of special important are lines which describe external variable assignments
(surrounded by "'s), loops (begin with lo), phases (contain ip of dp) or
increments (contain id, dd, ipu or dpu).  These lines are parsed when reading
the file with nmrglue.

"""

from functools import reduce
import os
from warnings import warn

import numpy as np

from . import fileiobase
from ..process import proc_base


# data creation

def create_data(data):
    """
    Create a bruker data array (recast into a complex128 or int32)
    """
    if np.iscomplexobj(data):
        return np.array(data, dtype='complex128')
    else:
        return np.array(data, dtype='int32')


# universal dictionary functions

def guess_udic(dic, data, strip_fake=False):
    """
    Guess parameters of universal dictionary from dic, data pair.

    Parameters
    ----------
    dic : dict
        Dictionary of Bruker parameters.
    data : ndarray
        Array of NMR data.
    strip_fake: bool
        If data is proceed (i.e. read using `bruker.read_pdata`) and the Bruker
        processing parameters STSI and/or STSR are set, the returned sweep
        width and carrier frequencies is changed to values that are incorrect
        but instead can are intended to trick the normal unit_conversion object
        into producing the correct result.

    Returns
    -------
    udic : dict
        Universal dictionary of spectral parameters.

    """
    # TODO if pprog, acqus, procs are in dic use them better.

    # create an empty universal dictionary
    udic = fileiobase.create_blank_udic(data.ndim)

    # update default values
    for b_dim in range(data.ndim):
        udic[b_dim]["size"] = data.shape[b_dim]

        # try to add additional parameter from acqus dictionary keys
        try:
            add_axis_to_udic(udic, dic, b_dim, strip_fake)
        except:
            warn("Failed to determine udic parameters for dim: %i" % (b_dim))
    return udic


def add_axis_to_udic(udic, dic, udim, strip_fake):
    """
    Add axis parameters to a udic.

    Parameters
    ----------
    udic : dict
        Universal dictionary to update, modified in place.
    dic : dict
        Bruker dictionary used to determine axes parameters.
    dim : int
        Universal dictionary dimension to update.
    strip_fake: bool
        See `bruker.guess_udic`

    """
    # This could still use some work
    b_dim = udic['ndim'] - udim - 1  # last dim
    acq_file = "acqu" + str(b_dim + 1) + "s"
    pro_file = "proc" + str(b_dim + 1) + "s"

    # Because they're inconsistent,..
    if acq_file == "acqu1s":
        acq_file = "acqus"

    if pro_file == "proc1s":
        pro_file = "procs"

    if acq_file in dic:
        sw = dic[acq_file]["SW_h"]
    elif pro_file in dic:
        sw = dic[pro_file]["SW_p"]
        # procNs files store sw (in Hz) with the 'SW_p' key instead of 'SW_h'.
        # this is a bug in TopSpin (TopSpin3.5pl7)

    if acq_file in dic:
        udic[udim]["label"] = dic[acq_file]["NUC1"]
    elif pro_file in dic:
        udic[udim]["label"] = dic[pro_file]["AXNUC"]

    try:
        obs = dic[pro_file]["SF"]
        if acq_file in dic:
            car = (dic[acq_file]["SFO1"] - obs) * 1e6
        else:
            # we should be able to use the 'OFFSET' parameter in procNs to
            # calculate 'car'. But this is slightly off (~ 5E-3 Hz)
            # most likely because the procs file does not store the OFFSET
            # to a high precision. Hence the value in acquNs is given priority
            car = dic[pro_file]["OFFSET"]*obs - sw/2

    except KeyError:
        warn('The chemical shift referencing was not corrected for "sr".')
        obs = dic[acq_file]["SFO1"]
        car = dic[acq_file]["O1"]

    if strip_fake:
        try:
            # Temporary parameters
            w = sw/float(dic[pro_file]["FTSIZE"])
            d = (w * dic[pro_file]["STSR"]) + (w * dic[pro_file]["STSI"]/2.0)

            # Fake car frequency
            car -= (d-(sw/2.0))

            # Fake sw frequency
            sw = w * dic[pro_file]["STSI"]
        except KeyError:
            pass

    udic[udim]["sw"] = sw
    udic[udim]["car"] = car
    udic[udim]["obs"] = obs

    if acq_file in dic:
        if acq_file == "acqus":
            if dic['acqus']['AQ_mod'] == 0:     # qf
                udic[udim]['complex'] = False
            else:
                udic[udim]['complex'] = True
        else:
            aq_mod = dic[acq_file]["FnMODE"]
            if aq_mod == 0:
                udic[udim]["encoding"] = "undefined"
            elif aq_mod == 1:
                udic[udim]["encoding"] = "magnitude"  # qf
            elif aq_mod == 2:
                udic[udim]["encoding"] = "magnitude"  # qsec
            elif aq_mod == 3:
                udic[udim]["encoding"] = "tppi"
            elif aq_mod == 4:
                udic[udim]["encoding"] = "states"
            elif aq_mod == 5:
                udic[udim]["encoding"] = "states-tppi"  # states-tppi
            elif aq_mod == 6:
                udic[udim]["encoding"] = "echo-antiecho"  # echo-antiecho
    else:
        if pro_file == "procs":
            # this seems to have the 'MC2' parameter always set to 0
            # irrespective of what the actual data is
            udic[udim]["complex"] = "undefined"
        else:
            # these are only used when params in acquNs are 'undefined'
            # but in absence of acqus, this is the best that can be done
            aq_mod = dic[pro_file]["MC2"]
            if aq_mod == 0:
                udic[udim]["encoding"] = "magnitude"  # qf
            elif aq_mod == 1:
                udic[udim]["encoding"] = "magnitude"  # qsec
            elif aq_mod == 2:
                udic[udim]["encoding"] = "tppi"
            elif aq_mod == 3:
                udic[udim]["encoding"] = "states"
            elif aq_mod == 4:
                udic[udim]["encoding"] = "states-tppi"
            elif aq_mod == 5:
                udic[udim]["encoding"] = "echo-antiecho"

    return udic


def create_dic(udic):
    """
    Create a Bruker parameter dictionary from a universal dictionary.

    Parameters
    ----------
    udic : dict
        Universal dictionary of spectral parameters.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.

    """
    ndim = udic['ndim']

    # determine the size in bytes
    if udic[ndim - 1]["complex"]:
        bytes = 8
    else:
        bytes = 4

    for k in range(ndim):
        bytes *= udic[k]["size"]

    dic = {"FILE_SIZE": bytes}

    # create the pprog dictionary parameter
    dic["pprog"] = {'incr': [[], [1]] * (ndim * 2 - 2),
                    'loop': [2] * (ndim * 2 - 2),
                    'ph_extra': [[]] * (ndim * 2 - 2),
                    'phase': [[]] * (ndim * 2 - 2),
                    'var': {}}

    # create acqus dictionary parameters and fill in loop sizes
    dic['acqus'] = create_acqus_dic(udic[ndim - 1], direct=True)
    if ndim >= 2:
        dic["acqu2s"] = create_acqus_dic(udic[ndim - 2])
        dic["pprog"]["loop"][1] = udic[ndim - 2]["size"] // 2
    if ndim >= 3:
        dic["acqu3s"] = create_acqus_dic(udic[ndim - 3])
        dic["pprog"]["loop"][3] = udic[ndim - 3]["size"] // 2
    if ndim >= 4:
        dic["acqu4s"] = create_acqus_dic(udic[ndim - 4])
        dic["pprog"]["loop"][5] = udic[ndim - 4]["size"] // 2

    return dic


def create_acqus_dic(adic, direct=False):
    """
    Create a Bruker acqus dictionary from an Universal axis dictionary.
    Set direct=True for direct dimension.
    """
    if adic["complex"]:
        AQ_mod = 3
        if direct:
            TD = int(np.ceil(adic["size"] / 256.) * 256) * 2
        else:
            TD = adic["size"]
    else:
        AQ_mod = 1
        if direct:
            TD = int(np.ceil(adic["size"] / 256.) * 256)
        else:
            TD = adic["size"]

    s = '##NMRGLUE automatically created parameter file'
    return {'_comments': [], '_coreheader': [s], 'AQ_mod': AQ_mod, 'TD': TD}


# Global read/write function and related utilities

def read(dir=".", bin_file=None, acqus_files=None, pprog_file=None, shape=None,
         cplex=None, big=None, isfloat=None, read_pulseprogram=True,
         read_acqus=True, procs_files=None, read_procs=True):
    """
    Read Bruker files from a directory.

    Parameters
    ----------
    dir : str
        Directory to read from.
    bin_file : str, optional
        Filename of binary file in directory. None uses standard files.
    acqus_files : list, optional
        List of filename(s) of acqus parameter files in directory. None uses
        standard files.
    pprog_file : str, optional
        Filename of pulse program in directory. None uses standard files.
    shape : tuple, optional
        Shape of resulting data.  None will guess the shape from the spectral
        parameters.
    cplex : bool, optional
        True is direct dimension is complex, False otherwise. None will guess
        quadrature from spectral parameters.
    big : bool or None, optional
        Endianness of binary file. True for big-endian, False for
        little-endian, None to determine endianness from acqus file(s).
    isfloat : bool or None, optional
        Data type of binary file. True for float64, False for int32. None to
        determine data type from acqus file(s).
    read_pulseprogram : bool, optional
        True to read pulse program, False prevents reading.
    read_acqus : bool, optional
        True to read acqus files(s), False prevents reading.
    procs_files : list, optional
        List of filename(s) of procs parameter files in directory. None uses
        standard files.
    read_procs : bool, optional
        True to read procs files(s), False prevents reading.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    data : ndarray
        Array of NMR data.

    See Also
    --------
    read_pdata : Read Bruker processed files.
    read_lowmem : Low memory reading of Bruker files.
    write : Write Bruker files.

    """
    if os.path.isdir(dir) is not True:
        raise IOError("directory %s does not exist" % (dir))

    # Take a shot at reading the procs file
    if read_procs:
        dic = read_procs_file(dir, procs_files)
    else:
        # create an empty dictionary
        dic = dict()

    # determine parameter automatically
    if bin_file is None:
        if os.path.isfile(os.path.join(dir, "fid")):
            bin_file = "fid"
        elif os.path.isfile(os.path.join(dir, "ser")):
            bin_file = "ser"

        # Look two directory levels lower.
        elif os.path.isdir(os.path.dirname(os.path.dirname(dir))):

            # ! change the dir
            dir = os.path.dirname(os.path.dirname(dir))

            if os.path.isfile(os.path.join(dir, "fid")):
                bin_file = "fid"
            elif os.path.isfile(os.path.join(dir, "ser")):
                bin_file = "ser"
            else:
                mesg = "No Bruker binary file could be found in %s"
                raise IOError(mesg % (dir))
        else:
            mesg = "No Bruker binary file could be found in %s"
            raise IOError(mesg % (dir))

    if read_acqus:
        # read the acqus_files and add to the dictionary
        acqus_dic = read_acqus_file(dir, acqus_files)
        dic = _merge_dict(dic, acqus_dic)

    if pprog_file is None:
        pprog_file = "pulseprogram"

    # read the pulse program and add to the dictionary
    if read_pulseprogram:
        try:
            dic["pprog"] = read_pprog(os.path.join(dir, pprog_file))
        except:
            warn('Error reading the pulse program')

    # determine file size and add to the dictionary
    dic["FILE_SIZE"] = os.stat(os.path.join(dir, bin_file)).st_size

    # determine shape and complexity for direct dim if needed
    if shape is None or cplex is None:
        gshape, gcplex = guess_shape(dic)
        if gcplex is True:    # divide last dim by 2 if complex
            t = list(gshape)
            t[-1] = t[-1] // 2
            gshape = tuple(t)
    if shape is None:
        shape = gshape
    if cplex is None:
        cplex = gcplex

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "acqus" in dic and "BYTORDA" in dic["acqus"]:
            if dic["acqus"]["BYTORDA"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "acqus" in dic and "DTYPA" in dic["acqus"]:
            if dic["acqus"]["DTYPA"] == 2:
                isfloat = True
            else:
                isfloat = False

    # read the binary file
    f = os.path.join(dir, bin_file)
    null, data = read_binary(f, shape=shape, cplex=cplex, big=big,
                             isfloat=isfloat)
    return dic, data


def read_lowmem(dir=".", bin_file=None, acqus_files=None, pprog_file=None,
                shape=None, cplex=None, big=None, isfloat=None,
                read_pulseprogram=True, read_acqus=True, procs_files=None,
                read_procs=True):
    """
    Read Bruker files from a directory using minimal amounts of memory.

    See :py:func:`read` for Parameters.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    data : array_like
        Low memory object which can access NMR data on demand.

    See Also
    --------
    read : Read Bruker files.
    write_lowmem : Write Bruker files using minimal amounts of memory.

    """

    if os.path.isdir(dir) is not True:
        raise IOError("directory %s does not exist" % (dir))

    # Take a shot at reading the procs file
    if read_procs:
        dic = read_procs_file(dir, procs_files)
    else:
        # create an empty dictionary
        dic = dict()

    # determine parameter automatically
    if bin_file is None:
        if os.path.isfile(os.path.join(dir, "fid")):
            bin_file = "fid"
        elif os.path.isfile(os.path.join(dir, "ser")):
            bin_file = "ser"

        # Look two directory levels lower.
        elif os.path.isdir(os.path.dirname(os.path.dirname(dir))):

            # ! change the dir
            dir = os.path.dirname(os.path.dirname(dir))

            if os.path.isfile(os.path.join(dir, "fid")):
                bin_file = "fid"
            elif os.path.isfile(os.path.join(dir, "ser")):
                bin_file = "ser"
            else:
                mesg = "No Bruker binary file could be found in %s"
                raise IOError(mesg % (dir))
        else:
            mesg = "No Bruker binary file could be found in %s"
            raise IOError(mesg % (dir))

    if read_acqus:
        # read the acqus_files and add to the dictionary
        acqus_dic = read_acqus_file(dir, acqus_files)
        dic = _merge_dict(dic, acqus_dic)

    if pprog_file is None:
        pprog_file = "pulseprogram"

    # read the pulse program and add to the dictionary
    if read_pulseprogram:
        dic["pprog"] = read_pprog(os.path.join(dir, pprog_file))

    # determine file size and add to the dictionary
    dic["FILE_SIZE"] = os.stat(os.path.join(dir, bin_file)).st_size

    # determine shape and complexity for direct dim if needed
    if shape is None or cplex is None:
        gshape, gcplex = guess_shape(dic)
        if gcplex is True:    # divide last dim by 2 if complex
            t = list(gshape)
            t[-1] = t[-1] // 2
            gshape = tuple(t)
    if shape is None:
        shape = gshape
    if cplex is None:
        cplex = gcplex

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "acqus" in dic and "BYTORDA" in dic["acqus"]:
            if dic["acqus"]["BYTORDA"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "acqus" in dic and "DTYPA" in dic["acqus"]:
            if dic["acqus"]["DTYPA"] == 2:
                isfloat = True
            else:
                isfloat = False

    # read the binary file
    f = os.path.join(dir, bin_file)
    null, data = read_binary_lowmem(f, shape=shape, cplex=cplex, big=big,
                                    isfloat=isfloat)
    return dic, data


def read_acqus_file(dir='.', acqus_files=None):
    """
    Read Bruker acquisition files from a directory.

    Parameters
    ----------
    dir : str
        Directory to read from.
    acqus_files : list, optional
        List of filename(s) of acqus parameter files in directory. None uses
        standard files.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    """
    if acqus_files is None:
        acqus_files = []
        for f in ["acqus", "acqu2s", "acqu3s", "acqu4s"]:
            if os.path.isfile(os.path.join(dir, f)):
                acqus_files.append(f)

    # create an empty dictionary
    dic = dict()

    # read the acqus_files and add to the dictionary
    for f in acqus_files:
        dic[f] = read_jcamp(os.path.join(dir, f))
    return dic


def read_procs_file(dir='.', procs_files=None):
    """
    Read Bruker processing files from a directory.

    Parameters
    ----------
    dir : str
        Directory to read from.
    procs_files : list, optional
        List of filename(s) of procs parameter files in directory. None uses
        standard files.

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    """
    if procs_files is None:
        procs_files = []
        for f in ["procs", "proc2s", "proc3s", "proc4s"]:
            if os.path.isfile(os.path.join(dir, f)):
                procs_files.append(f)
        pdata_path = dir

    if procs_files == []:
        if os.path.isdir(os.path.join(dir, 'pdata')):
            pdata_folders = [folder for folder in 
                             os.walk(os.path.join(dir, 'pdata'))][0][1]
            if '1' in pdata_folders:
                pdata_path = os.path.join(dir, 'pdata', '1')
            else:
                pdata_path = os.path.join(dir, 'pdata', pdata_folders[0]) 
                
    for f in ["procs", "proc2s", "proc3s", "proc4s"]:
            if os.path.isfile(os.path.join(pdata_path, f)):
                procs_files.append(f)

    # create an empty dictionary
    dic = dict()

    # read the acqus_files and add to the dictionary
    for f in procs_files:
        dic[f] = read_jcamp(os.path.join(pdata_path, f))
    return dic


def write(dir, dic, data, bin_file=None, acqus_files=None, procs_files=None,
          pprog_file=None, overwrite=False, big=None, isfloat=None, 
          write_prog=True, write_acqus=True, write_procs=False, 
          pdata_folder=False):
    """
    Write Bruker files to disk.

    Parameters
    ----------
    dir : str
        Directory to write files to.
    dir : dict
        Dictionary of Bruker parameters.
    data : array_like
        Array of NMR data
    bin_file : str, optional
        Filename of binary file in directory. None uses standard files.
    acqus_files : list, optional
        List of filename(s) of acqus parameter files in directory. None uses
        standard files.
    procs_file : list, optional
        List of filename(s) of procs parameter files (to write out). None uses a
        list of standard files
    pprog_file : str, optional
        Filename of pulse program in directory. None uses standard files.
    overwrite : bool, optional
        Set True to overwrite files, False will raise a Warning if files
        exist.
    big : bool or None, optional
        Endianness of binary file. True for big-endian, False for
        little-endian, None to determine endianness from Bruker dictionary.
    isfloat : bool or None, optional
        Data type of binary file. True for float64, False for int32. None to
        determine data type from Bruker dictionary.
    write_pprog : bool, optional
        True to write the pulse program file, False prevents writing.
    write_acqus : bool, optional
        True to write the acqus files(s), False prevents writing.
    write_procs : bool, optional
        True to write the procs files(s), False prevents writing.
    pdata_folder : int, optional
        Makes a folder and a subfolder ('pdata/pdata_folder') inside the given
        directory where pdata_folder is an integer. procN and procNs files are 
        stored inside pdata_folder. pdata_folder=False (or =0) does not make the 
        pdata folder and pdata_folder=True makes folder '1'.

    See Also
    --------
    write_lowmem : Write Bruker files using minimal amounts of memory.
    read : Read Bruker files.

    """
    # determine parameters automatically
    if bin_file is None:
        if data.ndim == 1:
            bin_file = "fid"
        else:
            bin_file = "ser"

    if acqus_files is None:
        acq = ["acqus", "acqu2s", "acqu3s", "acqu4s"]
        acqus_files = [k for k in acq if (k in dic)]
        acqu_files = [k[:-1] for k in acqus_files]

    if procs_files is None:
        proc = ["procs", "proc2s", "proc3s", "proc4s"]
        procs_files = [k for k in proc if (k in dic)]
        proc_files = [k[:-1] for k in procs_files]

    if pprog_file is None:
        pprog_file = "pulseprogram"

    # write out the acqus files
    if write_acqus:
        for f in acqus_files:
            write_jcamp(dic[f], os.path.join(dir, f), overwrite=overwrite)
        for f in acqu_files:
            write_jcamp(dic[f+'s'], os.path.join(dir, f), overwrite=overwrite)

    # write out the procs files
    if write_procs:
        if pdata_folder is not False:
            try:
                procno = str(int(pdata_folder))
                pdata_path = os.path.join(dir, 'pdata', procno)
            except ValueError:
                raise ValueError('pdata_folder should be an integer')
            
            if not os.path.isdir(pdata_path):
                os.makedirs(pdata_path)
        else:
            pdata_path = dir
    
        for f in procs_files:
            write_jcamp(dic[f], os.path.join(pdata_path, f))
        for f in proc_files:
            write_jcamp(dic[f+'s'], os.path.join(pdata_path, f))

    # write out the pulse program
    if write_prog:
        write_pprog(os.path.join(dir, pprog_file), dic["pprog"],
                    overwrite=overwrite)

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "acqus" in dic and "BYTORDA" in dic["acqus"]:
            if dic["acqus"]["BYTORDA"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "acqus" in dic and "DTYPA" in dic["acqus"]:
            if dic["acqus"]["DTYPA"] == 2:
                isfloat = True
            else:
                isfloat = False

    # write out the binary data
    bin_full = os.path.join(dir, bin_file)
    write_binary(bin_full, dic, data, big=big, isfloat=isfloat,
                 overwrite=overwrite)
    return


def write_lowmem(dir, dic, data, bin_file=None, acqus_files=None,
                 pprog_file=None, overwrite=False, big=None, isfloat=None,
                 write_prog=True, write_acqus=True):
    """
    Write Bruker files using minimal amounts of memory (trace by trace).

    See :py:func:`write` for Parameters.

    See Also
    --------
    write : Write Bruker files.
    read_lowmem : Read Bruker files using minimal amounts of memory.

    """
    # determine parameters automatically
    if bin_file is None:
        if data.ndim == 1:
            bin_file = "fid"
        else:
            bin_file = "ser"

    if acqus_files is None:
        acq = ["acqus", "acqu2s", "acqu3s", "acqu4s"]
        acqus_files = [k for k in acq if (k in dic)]

    if pprog_file is None:
        pprog_file = "pulseprogram"

    # write out the acqus files
    if write_acqus:
        for f in acqus_files:
            write_jcamp(dic[f], os.path.join(dir, f), overwrite=overwrite)

    # write out the pulse program
    if write_prog:
        write_pprog(os.path.join(dir, pprog_file), dic["pprog"],
                    overwrite=overwrite)

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "acqus" in dic and "BYTORDA" in dic["acqus"]:
            if dic["acqus"]["BYTORDA"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "acqus" in dic and "DTYPA" in dic["acqus"]:
            if dic["acqus"]["DTYPA"] == 2:
                isfloat = True
            else:
                isfloat = False

    # write out the binary data
    bin_full = os.path.join(dir, bin_file)
    write_binary_lowmem(bin_full, dic, data, big=big, isfloat=isfloat,
                        overwrite=overwrite)
    return


def write_pdata(dir, dic, data, shape=None, submatrix_shape=None, 
        bin_file=None, procs_files=None, write_procs=False, 
        pdata_folder=False, overwrite=False, big=None, isfloat=None, 
        restrict_access=True):
    """
    Write processed Bruker files to disk.

    Parameters
    ----------
    dir : str
        Directory to write files to.
    dic : dict
        Dictionary of Bruker parameters.
    data : array_like
        Array of NMR data
    shape : tuple, optional
        Shape of data, if file is to be written with a shape
        different than data.shape
    submatrix_shape : tuple, optional
        Shape of the submatrix used to store data (using Bruker specifications)
        If this is not given, the submatrix shape will be guessed from dic
    bin_file : str, optional
        Filename of binary file in directory. None uses standard files.
    procs_file : list, optional
        List of filename(s) of procs parameter files (to write out). None uses a
        list of standard files
    write_procs : Bool
        True to write out the procs files
    pdata_folder : int, optional
        Makes a folder and a subfolder ('pdata/pdata_folder') inside the given
        directory where pdata_folder is an integer. All files (procs and data) are 
        stored inside pdata_folder. pdata_folder=False (or =0) does not make the 
        pdata folder and pdata_folder=True makes folder '1'.
    overwrite : bool, optional
        Set True to overwrite files, False will raise a Warning if files
        exist.
    big : bool or None, optional
        Endianness of binary file. True for big-endian, False for
        little-endian, None to determine endianness from Bruker dictionary.
    isfloat : bool or None, optional
        Data type of binary file. True for float64, False for int32. None to
        determine data type from Bruker dictionary.
    restrict_access : not implemented

    """
   
    # see that data consists of only real elements
    data = data.real

    # scale and covert the data into int32 type
    data = array_to_int(data)

    # see if the dimensionality is given
    # else, set it to the dimensions of data
    if shape == None:
        shape = data.shape

    # guess data dimensionality
    ndim = len(shape)

    # update PARMODE in dictionary
    # This is required when writing back 1D slices from a 2D, 2D planes of 3D, etc
    dic['procs']['PPARMOD'] = ndim - 1

    # reorder the submatrix according 
    if submatrix_shape is None:
        submatrix_shape = guess_shape_and_submatrix_shape(dic)[1]

    data = reorder_submatrix(data, shape, submatrix_shape, reverse=True)

    # see if pdata_folder needs to make and set write path
    if pdata_folder is not False:
        try:
            procno = str(int(pdata_folder))
            pdata_path = os.path.join(dir, 'pdata', procno)
        except ValueError:
            raise ValueError('pdata_folder should be an integer')

        if not os.path.isdir(pdata_path):
            os.makedirs(pdata_path)
    else:
        pdata_path = dir

    # write out the procs files only for the desired dimensions
    if write_procs:
        if procs_files == None:
            proc = ['procs'] + ['proc{}s'.format(i) for i in range(2, ndim+1)]
            procs_files = [f for f in proc if (f in dic)]
    
        for f in procs_files:
            write_jcamp(dic[f], os.path.join(pdata_path, f),
                    overwrite=overwrite)
            write_jcamp(dic[f], os.path.join(pdata_path, f[:-1]),
                    overwrite=overwrite)

    if bin_file == None:
        bin_file = str(ndim) + 'r'*ndim

    bin_full = os.path.join(pdata_path, bin_file)
    write_binary(bin_full, dic, data, big=big, isfloat=isfloat,
                 overwrite=overwrite)
    return
        

def guess_shape(dic):
    """
    Determine data shape and complexity from Bruker dictionary.

    Returns
    -------
    shape : tuple
        Shape of data in Bruker binary file (R+I for all dimensions).
    cplex : bool
        True for complex data in last (direct) dimension, False otherwise.

    """
    # determine complexity of last (direct) dimension
    try:
        aq_mod = dic["acqus"]["AQ_mod"]
    except KeyError:
        aq_mod = 0

    if aq_mod == 0 or aq_mod == 2:
        cplex = False
    elif aq_mod == 1 or aq_mod == 3:
        cplex = True
    else:
        raise ValueError("Unknown Aquisition Mode")

    # file size
    try:
        fsize = dic["FILE_SIZE"]
    except KeyError:
        warn("cannot determine shape do to missing FILE_SIZE key")
        return (1,), True

    # extract td0,td1,td2,td3 from dictionaries
    try:
        td0 = float(dic["acqus"]["TD"])
    except KeyError:
        td0 = 1024   # default value

    try:
        td2 = int(dic["acqu2s"]["TD"])
    except KeyError:
        td2 = 0     # default value

    try:
        td1 = float(dic["acqu3s"]["TD"])
    except KeyError:
        td1 = int(td2)   # default value

    try:
        td3 = int(dic["acqu4s"]["TD"])
    except KeyError:
        td3 = int(td1)     # default value

    # From the acquisition reference manual (section on parameter NBL):
    #     ---
    #     If TD is not a multiple of 256 (1024 bytes), successive FIDs will
    #     still begin at 1024 byte memory boundaries. This is so for the FIDs
    #     in the acquisition memory as well as on disk. The size of the raw
    #     data file (ser) is therefore always a multiple of 1024 times NBL.
    #     ---
    # This seems to hold for 1D data sets as well. However, this paragraph
    # assumes that each data point is 4 bytes, hence the "multiple of 256".
    # For data in DTYPA=2 (float64), each point is 8 bytes, so while it always
    # allocates the fids in 1024-byte blocks, for float64 data it pads the data
    # (by points) out to multiples of 128, not 256. So we need to get the
    # data type before we guess the shape of the last dimension.

    # extract data type from dictionary
    try:
        dtypa = int(dic["acqus"]["DTYPA"])
    except KeyError:
        dtypa = 0   # default value, int32 data

    # last (direct) dimension is given by "TD" parameter in acqus file
    # rounded up to nearest (1024/(bytes per point))
    # next-to-last dimension may be given by "TD" in acqu2s. In 3D+ data
    # this is often the sum of the indirect dimensions
    if dtypa == 2:
        shape = [0, 0, td2, int(np.ceil(td0 / 128.) * 128.)]
    else:
        shape = [0, 0, td2, int(np.ceil(td0 / 256.) * 256.)]

    # additional dimension given by data size
    if shape[2] != 0 and shape[3] != 0:
        shape[1] = fsize // (shape[3] * shape[2] * 4)
        shape[0] = fsize // (shape[3] * shape[2] * shape[1] * 4)

    # if there in no pulse program parameters in dictionary return currect
    # shape after removing zeros
    if "pprog" not in dic or "loop" not in dic["pprog"]:
        return tuple([int(i) for i in shape if i > 1]), cplex

    # if pulseprogram dictionary is missing loop or incr return current shape
    pprog = dic["pprog"]
    if "loop" not in pprog or "incr" not in pprog:
        return tuple([int(i) for i in shape if i > 1]), cplex

    # determine indirect dimension sizes from pulseprogram parameters
    loop = pprog["loop"]
    loopn = len(loop)       # number of loops
    li = [len(i) for i in pprog["incr"]]    # length of incr lists

    # replace td0,td1,td2,td3 in loop list
    rep = {'td0': td0, 'td1': td1, 'td2': td2, 'td3': td3}
    for i, v in enumerate(loop):
        if v in rep.keys():
            loop[i] = rep[v]

    # if the loop variables contains strings, return current shape
    # these variables could be resolved from the var key in the pprog dict
    # but this would require executing unknown code to perform the
    # arithmetic present in the string.
    if str in [type(e) for e in loop]:
        return tuple([int(i) for i in shape if i > 1]), cplex

    # size of indirect dimensions based on number of loops in pulse program
    # there are two kinds of loops, active and passive.
    # active loops are from indirect dimension increments, the corresponding
    # incr lists should have non-zero length and the size of the dimension
    # is twice that of the active loop size.
    # passive loops are from phase cycles and similar elements, these should
    # have zero length incr lists and should be of length 2.

    # The following checks for these and updates the indirect dimension
    # if the above is found.
    if loopn == 1:    # 2D with no leading passive loops
        if li[0] != 0:
            shape[2] = loop[0]
            shape = shape[-2:]

    elif loopn == 2:  # 2D with one leading passive loop
        if loop[0] == 2 and li[0] == 0 and li[1] != 0:
            shape[2] = 2 * loop[1]
            shape = shape[-2:]

    elif loopn == 3:  # 2D with two leading passive loops
        if (loop[0] == 2 and loop[1] == 2 and li[0] == 0 and li[1] == 0
                and li[2] != 0):
            shape[2] = 2 * loop[2]
            shape = shape[-2:]

    elif loopn == 4:  # 3D with one leading passive loop for each indirect dim
        if loop[0] == 2 and li[0] == 0 and li[1] != 0:
            shape[2] = 2 * loop[1]
        if loop[2] == 2 and li[2] == 0 and li[3] != 0:
            shape[1] = 2 * loop[3]
            shape = shape[-3:]

    elif loopn == 5:  # 3D with two/one leading passive loops
        if loop[1] == 2 and li[0] == 0 and li[1] == 0 and li[2] != 0:
            shape[2] = 2 * loop[2]
        if loop[3] == 2 and li[0] == 0 and li[3] == 0 and li[4] != 0:
            shape[1] = 2 * loop[4]
            shape = shape[-3:]

    elif loopn == 6:  # 4D with one leading passive loop for each indirect dim
        if loop[0] == 2 and li[0] == 0 and li[1] != 0:
            shape[2] = 2 * loop[1]
        if loop[2] == 2 and li[2] == 0 and li[3] != 0:
            shape[1] = 2 * loop[3]
        if loop[4] == 2 and li[4] == 0 and li[5] != 0:
            shape[0] = 2 * loop[5]

    elif loopn == 7:
        if loop[1] == 2 and li[0] == 0 and li[1] == 0 and li[2] != 0:
            shape[2] = 2 * loop[2]
        if loop[3] == 2 and li[0] == 0 and li[3] == 0 and li[4] != 0:
            shape[1] = 2 * loop[4]
        if loop[5] == 2 and li[0] == 0 and li[5] == 0 and li[6] != 0:
            shape[0] = 2 * loop[6]

    return tuple([int(i) for i in shape if i >= 2]), cplex


# Bruker processed binary (1r, 1i, 2rr, 2ri, etc) reading

def read_pdata(dir=".", bin_files=None, procs_files=None, read_procs=True,
               acqus_files=None, read_acqus=True, scale_data=True, shape=None,
               submatrix_shape=None, all_components=False, big=None,
               isfloat=None):
    """
    Read processed Bruker files from a directory.

    In Topspin and other programs this data is typically scaled by dividing by
    2 ** -NC_proc where NC_proc is defined in the procs file.  This scaling
    can be accomplished by setting the scale_data parameter to True.

    Parameters
    ----------
    dir : str
        Directory to read from.
    bin_files : list of str, optional
        List of filename of binary file in directory. None uses standard
        files.
    procs_files : list, optional
        List of filename(s) of procs parameter files in directory. None uses
        standard files.
    read_procs : bool, optional
        True to read procs files(s), False prevents reading.
    acqus_files : list, optional
        List of filename(s) of acqus parameter files in directory. None uses
        standard files.
    read_acqus : bool, optional
        True to read acqus files(s), False prevents reading.
    scale_data : bool, optional
        True, the default, to apply scaling defined in the procs file.  The
        data should almost always be scaled. False, returns the
        data as it appears in the file.
    shape : tuple, optional
        Shape of resulting data.  None will guess the shape from the
        parameters in the procs file(s).
    submatrix_shape : tuple, optional
        Shape of submatrix for 2D+ data.  None will guess the shape from
        the metadata in the procs file(s).
    all_components : bool
        True to return a list of all components, False returns just the
        all real component (1r, 2rr, 3rrr, etc).
    big : bool or None, optional
        Endianness of binary file. True for big-endian, False for
        little-endian, None to determine endianness from procs file(s).
    isfloat : bool or None, optional
        Data type of binary file. True for float64, False for int32. None to
        determine data type from procs file(s).

    Returns
    -------
    dic : dict
        Dictionary of Bruker parameters.
    data : ndarray or list
        Array of NMR data.  If all_components is True this is a list of array
        with each quadrature component.

    Notes
    -----
    There is currently no support for writing Bruker processed files or
    reading processed files using minimal memory.

    """
    # TODO read_pdata_lowmem, write_pdata

    if os.path.isdir(dir) is not True:
        raise IOError("directory %s does not exist" % (dir))

    # find binary files
    if bin_files is None:
        if os.path.isfile(os.path.join(dir, "1r")):
            if all_components:
                bin_files = ['1r', '1i']
            else:
                bin_files = ['1r']
        elif os.path.isfile(os.path.join(dir, "2rr")):
            if all_components:
                bin_files = ['2rr', '2ri', '2ir', '2ii']
            else:
                bin_files = ['2rr']
        elif os.path.isfile(os.path.join(dir, "3rrr")):
            if all_components:
                bin_files = ['3rrr', '3rri', '3rir', '3rii', '3irr', '3iri',
                             '3iir', '3iii']
            else:
                bin_files = ['3rrr']
        else:
            raise IOError("No Bruker binary file could be found in %s" % (dir))

    if read_procs:
        # read the procs_files and add to the dictionary
        dic = read_procs_file(dir, procs_files)
    else:
        # create an empty dictionary
        dic = dict()

    if read_acqus:
        # If acqus files were not listed check in the usual place.
        acqus_dir = os.path.dirname(os.path.dirname(dir))
        if acqus_files is not None:
            acqus_dic = read_acqus_file(dir, acqus_files)
            # Merge the two dicts.
            dic = _merge_dict(dic, acqus_dic)

        elif os.path.isdir(acqus_dir):
            acqus_dic = read_acqus_file(acqus_dir)
            # Merge the two dicts.
            dic = _merge_dict(dic, acqus_dic)

    # determine shape and complexity for direct dim if needed
    if submatrix_shape is None or shape is None:
        g_shape, g_submatrix_shape = guess_shape_and_submatrix_shape(dic)
        if shape is None:
            shape = g_shape
        if submatrix_shape is None:
            submatrix_shape = g_submatrix_shape

    # issue a warning is submatrix_shape or shape are still None
    if submatrix_shape is None:
        warn('Submatrix shape not defined, returning 1D data')
    if shape is None:
        warn('Data shape not defined, returning 1D data')

    # determine endianness (assume little-endian unless BYTORDA is 1)
    if big is None:
        big = False     # default value
        if "procs" in dic and "BYTORDP" in dic["procs"]:
            if dic["procs"]["BYTORDP"] == 1:
                big = True
            else:
                big = False

    # determine data type (assume int32 unless DTYPA is 2)
    if isfloat is None:
        isfloat = False     # default value
        if "procs" in dic and "DTYPP" in dic["procs"]:
            if dic["procs"]["DTYPP"] == 2:
                isfloat = True
            else:
                isfloat = False

    # read the binary file
    data = [read_pdata_binary(os.path.join(dir, f),
                              shape, submatrix_shape, big, isfloat)[1]
            for f in bin_files]

    # scale data if requested
    if scale_data:
        data = [scale_pdata(dic, d) for d in data]

    if len(data) == 1:
        return dic, data[0]
    else:
        return dic, data


def scale_pdata(dic, data): 
    """
    Scale Bruker processed data using parameters from the procs file.

    Parameters
    ----------
    dic : dict
        Dictionary of Bruker parameters.
{'fraction_non_alphanumeric': 0.09831129576248439, 'fraction_numerical': 0.011670381738186656, 'mean_word_length': 2.062911066628539, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7187104', 'n_tokens_mistral': 27193, 'n_tokens_neox': 26222, 'n_words': 9169}