<commit_before>import spanlib_fort
import MV
import Numeric


def stackData(*data):
    """ Takes several data files, of same time and stacks them up together """
    len_time=None
    axes=[]
    dout=None # data output
    for d in data:
        t=d.getTime()
        if t is None:
            raise 'Error, all data muist have a time dimension'
        if len_time is None:
            len_time=len(t)
        elif len_time!=len(t):
            raise 'Error all datasets must have the same time length!!!!'

        if d.getAxis(0)!=t:
            d=d(order='t...')

        axes.append(d.getAxisList())
        tdata,w,m=pack(d)
        if dout is None:
            dout=tdata
            weights=w
            mask=[m]
        else:
            print dout.shape,tdata.shape
            dout=Numeric.concatenate((dout,tdata))
            weights=Numeric.concatenate((weights,w))
            mask.append(m)

    return Numeric.transpose(dout),weights,mask,axes

def unStackData(din,weights,mask,axes):
    """Unstack data in the form returned from stackData"""
    print din.shape,mask,'888888888888888888888888888888'
    nvar=len(axes)

    if nvar!=len(mask):
        raise 'Error mask and var length not compatible'

    totsize=0
    for m in mask:
        totsize+=int(MV.sum(MV.ravel(m)))
    if totsize!=din.shape[1]:
        raise 'Error data and masks are not compatible in length!!!! (%s) and (%s)' % (totsize,din.shape[1])

    istart=0
    out=[]
    for i in range(nvar):
        m=mask[i]
        mlen=int(MV.sum(MV.ravel(m)))
        iend=istart+mlen
        data=Numeric.transpose(din[:,istart:iend])
        w=weights[istart:iend]
        ns1=len(axes[i][-1])
        ns2=len(axes[i][-2])
        print 'm',m
        print 'ns1,ns2',ns1,ns2
        print 'data.shape',data.shape
        print 'data',data
        print 'mlen',mlen
        up=spanlib_fort.unpack3d(m,ns1,ns2,data.shape[1],data,mlen,1.e20)
        unpacked = MV.transpose(MV.array(up))
        unpacked.setAxisList(axes[i])
        istart+=mlen
        out.append(unpacked)
    return out


def pack(data,weights=None):
    """ Computes weights and mask"""
    if Numeric.rank(data)==2: # Already packed but then vneeds weights!
        if weights is None:
            raise 'Error packed data must be sent with weights!'
        else:
            return Numeric.transpose(data),weights,True
    sh=list(data.shape)
    ns1=sh[-1]
    ns2=sh[-2]
    nt=sh[0]
    ## Weights ?
    if weights is None:
        try:
            import cdutil
            tmp=data
            while tmp.rank()>2:
                tmp=tmp[0]
            weights=cdutil.area_weights(tmp).raw_data()
            del(tmp)
        except:
            weights=MV.ones((sh[-2],sh[-1]),typecode='f')

    ## Now masking part
    ## First from data
    mask = data.mask()
    if mask is None:
        mask=MV.zeros(sh,typecode='f')

    if data.rank()>2:
        mask=MV.sum(mask,axis=0)

    ## Now add the ones from the weights
    print mask.shape,weights.shape
    mask=mask.filled()+MV.equal(weights,0.).filled(1)
    print 'After:',mask.shape

    ## >=1 means masked, Fortran "mask": 1 means data ==> 1-mask
    mask=1.-MV.greater_equal(mask,1).filled()
    mask=Numeric.transpose(mask)

    ## Number of points in spatial dimension
    ns=int(MV.sum(Numeric.ravel(mask)))

    ## Ok now calls fortran, but need to transpose first<commit_msg>example qui marche de stacking/unstacking, enlever des print<commit_after>import spanlib_fort
import MV
import Numeric


def stackData(*data):
    """ Takes several data files, of same time and stacks them up together """
    len_time=None
    axes=[]
    dout=None # data output
    for d in data:
        t=d.getTime()
        if t is None:
            raise 'Error, all data muist have a time dimension'
        if len_time is None:
            len_time=len(t)
        elif len_time!=len(t):
            raise 'Error all datasets must have the same time length!!!!'

        if d.getAxis(0)!=t:
            d=d(order='t...')

        axes.append(d.getAxisList())
        tdata,w,m=pack(d)
        if dout is None:
            dout=tdata
            weights=w
            mask=[m]
        else:
##             print dout.shape,tdata.shape
            dout=Numeric.concatenate((dout,tdata))
            weights=Numeric.concatenate((weights,w))
            mask.append(m)

    return Numeric.transpose(dout),weights,mask,axes

def unStackData(din,weights,mask,axes):
    """Unstack data in the form returned from stackData"""
##     print din.shape,mask,'888888888888888888888888888888'
    nvar=len(axes)

    if nvar!=len(mask):
        raise 'Error mask and var length not compatible'

    totsize=0
    for m in mask:
        totsize+=int(MV.sum(MV.ravel(m)))
    if totsize!=din.shape[1]:
        raise 'Error data and masks are not compatible in length!!!! (%s) and (%s)' % (totsize,din.shape[1])

    istart=0
    out=[]
    for i in range(nvar):
        m=mask[i]
        mlen=int(MV.sum(MV.ravel(m)))
        iend=istart+mlen
        data=Numeric.transpose(din[:,istart:iend])
        w=weights[istart:iend]
        ns1=len(axes[i][-1])
        ns2=len(axes[i][-2])
##         print 'm',m
##         print 'ns1,ns2',ns1,ns2
##         print 'data.shape',data.shape
##         print 'data',data
##         print 'mlen',mlen
        up=spanlib_fort.unpack3d(m,ns1,ns2,data.shape[1],data,mlen,1.e20)
        unpacked = MV.transpose(MV.array(up))
        unpacked.setAxisList(axes[i])
        istart+=mlen
        out.append(unpacked)
    return out


def pack(data,weights=None):
    """ Computes weights and mask"""
    if Numeric.rank(data)==2: # Already packed but then vneeds weights!
        if weights is None:
            raise 'Error packed data must be sent with weights!'
        else:
            return Numeric.transpose(data),weights,True
    sh=list(data.shape)
    ns1=sh[-1]
    ns2=sh[-2]
    nt=sh[0]
    ## Weights ?
    if weights is None:
        try:
            import cdutil
            tmp=data
            while tmp.rank()>2:
                tmp=tmp[0]
            weights=cdutil.area_weights(tmp).raw_data()
            del(tmp)
        except:
            weights=MV.ones((sh[-2],sh[-1]),typecode='f')

    ## Now masking part
    ## First from data
    mask = data.mask()
    if mask is None:
        mask=MV.zeros(sh,typecode='f')

    if data.rank()>2:
        mask=MV.sum(mask,axis=0)

    ## Now add the ones from the weights
##     print mask.shape,weights.shape
    mask=mask.filled()+MV.equal(weights,0.).filled(1)
##     print 'After:',mask.shape

    ## >=1 means masked, Fortran "mask": 1 means data ==> 1-mask
    mask=1.-MV.greater_equal(mask,1).filled()
    mask=Numeric.transpose(mask)

    ## Number of points in spatial dimension
    ns=int(MV.sum(Numeric.ravel(mask)))

    ## Ok now calls fortran, but need to transpose first
{'fraction_non_alphanumeric': 0.13164777680906714, 'fraction_numerical': 0.02005231037489102, 'mean_word_length': 2.280743565300286, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2446915', 'n_tokens_mistral': 2422, 'n_tokens_neox': 2290, 'n_words': 528}