<commit_before>#! /usr/bin/env python
"""
Functions involving masked arrays

Some functions are general array operations, others involve geospatial information

"""

import sys
import os
import glob

import numpy as np
from osgeo import gdal

from pygeotools.lib import iolib

#Notes on geoma
#Note: Need better init overloading
#http://stackoverflow.com/questions/141545/overloading-init-in-python
#Might make more sense to create ma subclass, and add gdal ds as new object
#http://stackoverflow.com/questions/12597827/how-to-subclass-numpy-ma-core-masked-array
#http://docs.scipy.org/doc/numpy/user/basics.subclassing.html

#Want to implement basic array indexing with map coordinates, display in plt

#See pyresample
#https://github.com/talltom/PyRaster/blob/master/rasterIO.py
#http://www2-pcmdi.llnl.gov/cdat/tutorials/training/cdat_2004/06-arrays-variables-etc.pdf
#http://docs.scipy.org/doc/numpy/user/basics.subclassing.html

#=======================
#Masked array stack
#=======================

#Want to add error attributes
#Want to make consistent with stack_count vs count keywords/attributes
class DEMStack:
    def __init__(self, fn_list=[], stack_fn=None, outdir=None, res=None, extent=None, srs=None, trend=True, robust=False, med=False, stats=True, save=True, sort=True, datestack=True, mask_geom=None, min_dt_ptp=np.nan, n_thresh=2, n_cpu=None):
        self.sort = sort
        if self.sort:
            #This sorts filenames, should probably sort by datetime to be safe
            fn_list = sorted(fn_list, key=lambda x: os.path.split(x)[-1])
        self.fn_list = list(fn_list)
        self.stack_fn = stack_fn
        if not self.fn_list and stack_fn is None:
            raise ValueError('Must specify input filename list or existing stack filename')
        self.res = res
        self.extent = extent
        self.srs = srs 
        self.trend = trend
        self.robust = robust
        self.med = med
        self.stats = stats
        self.save = save
        self.datestack = datestack
        self.mask_geom = mask_geom
        #This is the minimum number of arrays in stack to compute trend
        self.n_thresh = n_thresh 
        #This is the minimum number of days between first and last timestamp to compute trend
        self.min_dt_ptp = min_dt_ptp
        self.n_cpu = n_cpu

        #Use this to limit memory use and filesizes
        #self.dtype = 'float32'
        self.dtype = np.float32

        #Want to do this before filenames, etc. are determined
        #self.get_date_list() 
        #if sort:
        #    idx = np.argsort(self.date_list_o)
        #    self.date_list = self.date_list[idx]
        #    self.date_list_o = self.date_list_o[idx]
        #    self.fn_list = (np.array(self.fn_list)[idx]).tolist()

        #Determine appropriate stack filename
        if outdir is None:
            #Use directory of first filename
            if self.fn_list:
                self.outdir = os.path.abspath(os.path.split(self.fn_list[0])[0])
            else:
                self.outdir = os.getcwd()
        else:
            if os.path.exists(outdir):
                self.outdir = outdir
            else:
                os.makedirs(outdir)
                self.outdir = outdir
                #raise IOError('Specified output directory does not exist')

        #If we're on Pleiades, make sure striping is set up properly on output directory
        #This now has check for lustre filesystem
        iolib.setstripe(self.outdir, self.n_cpu)

        #Flag specifying whether user has specified output stack filename
        #Hack to prevent new stack_fn generation if files are missing or sorted
        self.user_stack_fn = False
        if self.stack_fn is not None:
            self.user_stack_fn = True 

        if not self.user_stack_fn:
            if self.fn_list:
                self.get_stack_fn()
            else:
                raise ValueError('Must specify input filename list or existing stack filename')

        if os.path.exists(self.stack_fn):
            self.loadstack()
            #This was an attempt to ensure that input fn/res/extent is consistent with saved npz
            #Should really check to see if new extent falls within existing extent
            #Only regenerate if new extent is larger
            #Res check is not working correctly
            #print extent, self.extent
            #print res, self.res
            #if (self.extent != extent) or (self.res != res):
                #self.res = res
                #self.extent = extent
                #self.makestack()
                #if self.stats: 
                    #self.compute_stats()
                    #self.write_stats()
                #self.savestack()
        else:<commit_msg>malib: start cleaning up DEMStack outdir and stack_fn handling<commit_after>#! /usr/bin/env python
"""
Functions involving masked arrays

Some functions are general array operations, others involve geospatial information

"""

import sys
import os
import glob

import numpy as np
from osgeo import gdal

from pygeotools.lib import iolib

#Notes on geoma
#Note: Need better init overloading
#http://stackoverflow.com/questions/141545/overloading-init-in-python
#Might make more sense to create ma subclass, and add gdal ds as new object
#http://stackoverflow.com/questions/12597827/how-to-subclass-numpy-ma-core-masked-array
#http://docs.scipy.org/doc/numpy/user/basics.subclassing.html

#Want to implement basic array indexing with map coordinates, display in plt

#See pyresample
#https://github.com/talltom/PyRaster/blob/master/rasterIO.py
#http://www2-pcmdi.llnl.gov/cdat/tutorials/training/cdat_2004/06-arrays-variables-etc.pdf
#http://docs.scipy.org/doc/numpy/user/basics.subclassing.html

#=======================
#Masked array stack
#=======================

#Want to add error attributes
#Want to make consistent with stack_count vs count keywords/attributes
class DEMStack:
    def __init__(self, fn_list=[], stack_fn=None, outdir=None, res=None, extent=None, srs=None, trend=True, robust=False, med=False, stats=True, save=True, sort=True, datestack=True, mask_geom=None, min_dt_ptp=np.nan, n_thresh=2, n_cpu=None):
        self.sort = sort
        if self.sort:
            #This sorts filenames, should probably sort by datetime to be safe
            fn_list = sorted(fn_list, key=lambda x: os.path.split(x)[-1])
        self.fn_list = list(fn_list)
        self.stack_fn = stack_fn
        if not self.fn_list and stack_fn is None:
            raise ValueError('Must specify input filename list or existing stack filename')
        self.res = res
        self.extent = extent
        self.srs = srs 
        self.trend = trend
        self.robust = robust
        self.med = med
        self.stats = stats
        self.save = save
        self.datestack = datestack
        self.mask_geom = mask_geom
        #This is the minimum number of arrays in stack to compute trend
        self.n_thresh = n_thresh 
        #This is the minimum number of days between first and last timestamp to compute trend
        self.min_dt_ptp = min_dt_ptp
        self.n_cpu = n_cpu

        #Use this to limit memory use and filesizes
        #self.dtype = 'float32'
        self.dtype = np.float32

        #Want to do this before filenames, etc. are determined
        #self.get_date_list() 
        #if sort:
        #    idx = np.argsort(self.date_list_o)
        #    self.date_list = self.date_list[idx]
        #    self.date_list_o = self.date_list_o[idx]
        #    self.fn_list = (np.array(self.fn_list)[idx]).tolist()

        #TODO: cleanup the outdir and stack_fn handling

        #Determine appropriate stack filename
        if outdir is None:
            #Use directory of first filename
            if self.fn_list:
                self.outdir = os.path.abspath(os.path.split(self.fn_list[0])[0])
            else:
                self.outdir = os.getcwd()

        #Flag specifying whether user has specified output stack filename
        #Hack to prevent new stack_fn generation if files are missing or sorted
        self.user_stack_fn = False
        if self.stack_fn is not None:
            self.user_stack_fn = True 

        if not self.user_stack_fn:
            if self.fn_list:
                self.get_stack_fn()
            else:
                raise ValueError('Must specify input filename list or existing stack filename')

        if not os.path.exists(outdir):
            os.makedirs(outdir)
            self.outdir = outdir
            #raise IOError('Specified output directory does not exist')
        #If we're on Pleiades, make sure striping is set up properly on output directory
        #This now has check for lustre filesystem
        iolib.setstripe(self.outdir, self.n_cpu)

        if os.path.exists(self.stack_fn):
            self.loadstack()
            #This was an attempt to ensure that input fn/res/extent is consistent with saved npz
            #Should really check to see if new extent falls within existing extent
            #Only regenerate if new extent is larger
            #Res check is not working correctly
            #print extent, self.extent
            #print res, self.res
            #if (self.extent != extent) or (self.res != res):
                #self.res = res
                #self.extent = extent
                #self.makestack()
                #if self.stats: 
                    #self.compute_stats()
                    #self.write_stats()
                #self.savestack()
        else:
{'fraction_non_alphanumeric': 0.09167279218733593, 'fraction_numerical': 0.006090517694003991, 'mean_word_length': 2.3023578363384187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6308871', 'n_tokens_mistral': 2886, 'n_tokens_neox': 2765, 'n_words': 837}