<commit_before>
'''
Modules for Numerical Relativity Simulation Catalog:

* catalog: builds catalog given a cinfiguration file, or directory containing many configuration files.

* scentry: class for simulation catalog entry (should include io)

'''

#
from nrutils.core import settings as gconfig
from nrutils.core.basics import *
from nrutils.core import M_RELATIVE_SIGN_CONVENTION
import warnings,sys

# Class representation of configuration files. The contents of these files define where the metadata for each simulation is stored, and where the related NR data is stored.
class scconfig(smart_object):

    # Create scconfig object from configuration file location
    def __init__(this,config_file_location=None):

        # Required fields from smart_object
        this.source_file_path = []
        this.source_dir = []

        # call wrapper for constructor
        this.config_file_location = config_file_location
        this.reconfig()

    # The actual constructor: this will be called within utility functions so that scentry objects are configured with local settings.
    def reconfig(this):

        #
        if this.config_file_location is None:
            msg = '(!!) scconfig objects cannot be initialted/reconfigured without a defined "config_file_location" location property (i.e. string where the related config file lives)'
            raise ValueError(msg)

        # learn the contents of the configuration file
        this.learn_file( this.config_file_location, comment=[';','#'] )

        # validate the information learned from the configuration file against minimal standards
        this.valid = this.validate()

        # In some cases, it is useful to have this function return this
        return this

    # Validate the config file against a minimal set of required fields.
    def validate(this):

        # Import useful things
        from os.path import expanduser

        # Create a string with the current process name
        thisfun = inspect.stack()[0][3]

        # each scconfig object (and the realted file) MUST have the following attributes
        required_attrs = [ 'institute',                 # school or collaboration authoring run
                           'metadata_id',               # unique string that defines metadata files
                           'catalog_dir',               # local directory where all simulation folders are stored
                                                        # this directory allows catalog files to be portable
                           'data_file_name_format',     # formatting string for referencing l m and extraction parameter
                           'handler_location',          # location of python script which contains validator and
                                                        # learn_metadata functions
                           'is_extrapolated',           # users should set this to true if waveform is extrapolated
                                                        # to infinity
                           'default_par_list' ]   # list of default parameters for loading: default_extraction_parameter, default_level. NOTE that list must be of length 2

        # Make sure that each required attribute is a member of this objects dictionary representation. If it's not, throw an error.
        for attr in required_attrs:
            if not ( attr in this.__dict__ ):
                msg = '(!!) Error -- config file at %s does NOT contain required field %s' % (  magenta(this.config_file_location), attr )
                raise ValueError(msg)

        # Make sure that data_file_name_format is list of strings. The intention is to give the user the ability to define multiple formats for loading. For example, the GT dataset may have files that begin with Ylm_Weyl... and others that begin with mp_Weylscalar... .
        if isinstance( this.data_file_name_format, str ):
            this.data_file_name_format = [this.data_file_name_format]
        elif isinstance(this.data_file_name_format,list):
            for k in this.data_file_name_format:
                if not isinstance(k,str):
                    msg = '(!!) Error in %s: each element of data_file_name_format must be character not numeric. Found data_file_name_format = %s' % (magenta(this.config_file_location),k)
                    raise ValueError(msg)
                if len( k.split('%i') ) != 4:
                    msg = '(!!) Error in %s: All elements of data_file_name_format must have three integer formatting tags (%%i). The offending entry is %s.' % ( magenta(this.config_file_location), red(k) )
                    raise ValueError(msg)
        else:
            msg = '(!!)  Error in %s: data_file_name_format must be comma separated list.' %  magenta(this.config_file_location)

        # Make sure that catalog_dir is string
        if not isinstance( this.catalog_dir, str ):
            msg = 'catalog_dir values must be string'
            error(red(msg),thisfun)


        if 2 != len(this.default_par_list):
            msg = '(!!) Error in %s: default_par_list must be list containing default extraction parameter (Numeric value) and default level (also Numeric in value). Invalide case found: %s' % (magenta(this.config_file_location),list(this.default_par_list))
            raise ValueError(msg)

        # Make sure that all directories end with a forward slash
        for attr in this.__dict__:
            if 'dir' in attr:
                if this.__dict__[attr][-1] != '/':
                    this.__dict__[attr] += '/'
        # Make sure that user symbols (~) are expanded
        for attr in this.__dict__:
            if ('dir' in attr) or ('location' in attr):
                if isinstance(this.__dict__[attr],str):
                    this.__dict__[attr] = expanduser( this.__dict__[attr] )
                elif isinstance(this.__dict__[attr],list):
                    for k in this.__dict__[attr]:
                        if isinstance(k,str):
                            k = expanduser(k)

# Class for simulation catalog e.
class scentry:

    # Create scentry object given location of metadata file
    def __init__( this, config_obj, metadata_file_location ):

        # Keep an internal log for each scentry created
        this.log = '[Log for %s] The file is "%s".' % (this,metadata_file_location)

        # Store primary inputs as object attributes
        this.config = config_obj
        this.metadata_file_location = metadata_file_location

        # Validate the location of the metadata file: does it contain waveform information? is the file empty? etc
        this.is_valid = this.validate()

        # If valid, learn metadata. Note that metadata property are defined as none otherise. Also NOTE that the standard metadata is stored directly to this object's attributes.
        this.raw_metadata = None
        if this.is_valid is True:
            #
            print '## Working: %s' % cyan(metadata_file_location)
            this.log += ' This entry\'s metadata file is valid.'

            # i.e. learn the meta_data_file
            try:
                this.learn_metadata()
                this.label = sclabel( this )
            except:
                emsg = sys.exc_info()[1].message
                this.log += '%80s'%' [FATALERROR-1] The metadata failed to be read. There may be an external formatting inconsistency. It is being marked as invalid with None. The system says: %s'%emsg
                warning( 'The following error message will be logged: '+red(emsg),'scentry')
                this.is_valid = None # An external program may use this to do something
                this.label = 'invalid!'

        elif this.is_valid is False:
            print '## The following is '+red('invalid')+': %s' % cyan(metadata_file_location)
            this.log += ' This entry\'s metadta file is invalid.'

    #
    def validate(this):

        # determine the validity of the current object

        # import validation function given in config file
        from imp import load_source
        validator_module = load_source( '', this.config.handler_location )

        # Name the function representation that will be used to load the metadata file, and convert it to raw and standardized metadata
        validator = validator_module.validate

        # vet the directory where the metadata file lives for: waveform and additional metadata
        status = validator( this.metadata_file_location, config = this.config )

        #
        return status

    # Standardize metadata
    def learn_metadata(this):

        #
        from imp import load_source
        from numpy import allclose

        # import metadata converter functiongiven in config file
        converter_module = load_source( '', this.config.handler_location )

        # Name the function representation that will be used to load the metadata file, and convert it to raw and standardized metadata
        learn_institute_metadata = converter_module.learn_metadata

        # Eval and store standard metadata
        [standard_metadata, this.raw_metadata] = learn_institute_metadata( this.metadata_file_location )

        # Validate the standard metadata
        required_attrs = [ 'date_number',   # creation date (number!) of metadata file
                           'note',          # informational note relating to metadata
                           'madm',          # initial ADM mass = m1+m2 - initial binding energy
                           'b',             # initial orbital separation (scalar: M)
                           'R1', 'R2',      # initial component masses (scalars: M = m1+m2)
                           'm1', 'm2',      # initial component masses (scalars: M = m1+m2)
                           'P1', 'P2',      # initial component linear momenta (Vectors ~ M )
                           'L1', 'L2',      # initial component angular momental (Vectors ~ M)
                           'S1', 'S2',      # initial component spins (Vectors ~ M*M)
                           'mf', 'Sf',      # Final mass (~M) and final dimensionful spin (~M*M)
                           'extraction_parameter_is_radius', # If False, then raw data must be scaled by the extraction
                                                             # radius (it's ~ 1 ); else, raw data is not scaled by the
                                                             # extractino radius (it's ~ 1/r, and it will need to be
                                                             # scaled by r to then be proportional to 1).
                           'xf' ]           # Final dimensionless spin *Magnitude*: xf = |Sf|/(mf*mf)

        for attr in required_attrs:
            if attr not in standard_metadata.__dict__:
                msg = '(!!) Error -- Output of %s does NOT contain required field %s' % ( this.config.handler_location, attr )
                raise ValueError(msg)

        # Confer the required attributes to this object for ease of referencing
        for attr in standard_metadata.__dict__.keys():
            setattr( this, attr, standard_metadata.__dict__[attr] )

        # tag this entry with its inferred setname
        this.setname = this.raw_metadata.source_dir[-1].split( this.config.catalog_dir )[-1].split('/')[0]

        # tag this entry with the directory location of the metadata file. NOTE that the waveform data must be reference relative to this directory via config.data_file_name_format
        this.relative_simdir = this.raw_metadata.source_dir[-1].split( this.config.catalog_dir )[-1]

        # Basic sanity check for standard attributes. NOTE this section needs to be completed and perhaps externalized to the current function.

        # Check that initial binary separation is float
        if not isinstance( this.b , float ) :
            msg = 'b = %g' % this.b
            raise ValueError(msg)
        # Check that final mass is float
        if not isinstance( this.mf , float ) :
            msg = ''
            raise ValueError(msg)
        # Check that inital mass1 is float
        if not isinstance( this.m1 , float ) :
            msg = ''
            raise ValueError(msg)
        # Check that inital mass2 is float
        if not isinstance( this.m2 , float ) :
            msg = ''
            raise ValueError(msg)
        # Enfore m1>m2 convention.
        satisfies_massratio_convetion = lambda e: (not e.m1 > e.m2) and (not allclose(e.m1,e.m2,atol=1e-4))
        if satisfies_massratio_convetion(this):
            this.flip()
        if satisfies_massratio_convetion(this):
            msg = 'Mass ratio convention m1>m2 must be used. Check scentry.flip(). It should have corrected this! \n>> m1 = %g, m2 = %g' % (this.m1,this.m2)
            raise ValueError(msg)

    # Create dynamic function that references the user's current configuration to construct the simulation directory of this run.
    def simdir(this): return this.config.reconfig().catalog_dir + this.relative_simdir

    # Flip 1->2 associations.
    def flip(this):

        #
        from numpy import array,double

        # Store the flippoed variables to placeholders
        R1 = array(this.R2); R2 = array(this.R1);
        m1 = double(this.m2); m2 = double(this.m1);
        P1 = array(this.P2); P2 = array(this.P1);
        L1 = array(this.L2); L2 = array(this.L1);
        S1 = array(this.S2); S2 = array(this.S1);

        # Apply the flip to the current object
        this.R1 = R1; this.R2 = R2
        this.m1 = m1; this.m2 = m2
        this.P1 = P1; this.P2 = P2
        this.L1 = L1; this.L2 = L2
        this.S1 = S1; this.S2 = S2

    # Compare this scentry object to another using initial parameter fields. Return true false statement
    def compare2( this, that, atol=1e-3 ):

        #
        from numpy import allclose,hstack,double

        # Calculate an array of initial parameter values (the first element is 0 or 1 describing quasi-circularity)
        def param_array( entry ):

            # List of fields to add to array: initial parameters that are independent of initial separation
            field_list = [ 'm1', 'm2', 'S1', 'S2' ]

            #
            a = double( 'qc' in entry.label )
            for f in field_list:
                a = hstack( [a, entry.__dict__[f] ] )

            #
            return a

        # Perform comparison and return
        return allclose( param_array(this), param_array(that), atol=atol )



# Create the catalog database, and store it as a pickled file.
def scbuild(save=True):

    # Load useful packages
    from commands import getstatusoutput as bash
    from os.path import realpath, abspath, join, splitext, basename
    from os import pardir,system,popen
    import pickle

    # Create a string with the current process name
    thisfun = inspect.stack()[0][3]

    # Look for config files
    cpath_list = glob.glob( gconfig.config_path+'*.ini' )

    #
    if not cpath_list:
        msg = 'Cannot find configuration files (*.ini) in %s' % gconfig.config_path
        error(msg,thisfun)

    # Create config objects from list of config files
    configs = [ scconfig( config_path ) for config_path in cpath_list ]

    # For earch config
    for config in configs:

        # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #
        # Create streaming log file        #
        logfstr = gconfig.database_path + '/' + splitext(basename(config.config_file_location))[0] + '.log'
        msg = 'Opening log file in: '+cyan(logfstr)
        alert(msg,thisfun)
        logfid = open(logfstr, 'w')
        # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #

        # Search recurssively within the config's catalog_dir for files matching the config's metadata_id
        msg = 'Searching for %s in %s.' % ( cyan(config.metadata_id), cyan(config.catalog_dir) ) + yellow(' This may take a long time if the folder being searched is mounted from a remote drive.')
        alert(msg,thisfun)
        mdfile_list = rfind(config.catalog_dir,config.metadata_id,verbose=True)
        alert('done.',thisfun)

        # (try to) Create a catalog entry for each valid metadata file
        config.catalog = []
        h = -1
        for mdfile in mdfile_list:

            # Create tempoary scentry object
            entry = scentry(config,mdfile)

            # Write to the master log file
            h+=1
            logfid.write( '%5i\t%s\n'% (h,entry.log) )

            # If the obj is valid, add it to the catalog list, else ignore
            if entry.is_valid:
                config.catalog.append( entry )
            else:
                del entry

        # Store the catalog to the database_path
        if save:
            db = gconfig.database_path + '/' + splitext(basename(config.config_file_location))[0] + '.' + gconfig.database_ext
            with open(db, 'wb') as dbf:
                pickle.dump( config.catalog , dbf, pickle.HIGHEST_PROTOCOL )

        # Close the log file
        logfid.close()




# Function for searching through catalog files.
def scsearch( catalog = None,           # Manually input list of scentry objects to search through
              q = None,                 # RANGE of mass ratios (>=1) to search for
              nonspinning = None,       # Non-spinning initially
              spinaligned = None,       # spin-aligned with L AND no in-plane spin INITIALLY
              spinantialigned = None,   # spin-anti-aligned with L AND no in-plane spin INITIALLY
              precessing = None,        # not spin aligned
              notprecessing = None,     # not precessing
              equalspin = None,         # equal spin magnitudes
              unequalspin = None,       # not equal spin magnitudes
              antialigned = None,       # spin is in opposite direction of L
              setname = None,           # name of simulation set
              notsetname = None,        # list of setnames to ignore
              institute = None,         # list of institutes to accept
              keyword = None,           # list of keywords to accept (based on metadata directory string)
              notkeyword = None,        # list of keywords to not accept (based on metadata
                                        # directory string
              unique = None,            # if true, only simulations with unique initial conditions will be used
              plot = None,              # whether or not to show a plot of results
              verbose = None):          # be verbose

    # Print non None inputs to screen
    thisfun = inspect.stack()[0][3]
    if verbose is not None:
        for k in dir():
            if (eval(k) is not None) and (k != 'thisfun'):
                print '[%s]>> Found %s (=%r) keyword.' % (thisfun,textul(k),eval(k))

    '''
    Handle individual cases:
    '''

    #
    from os.path import realpath, abspath, join
    from os import pardir
    from numpy.linalg import norm
    from numpy import allclose,dot
    import pickle, glob

    # absolute tolerance for num comparisons
    tol = 1e-6

    # Handle the catalog input
    if catalog is None:
        # Get a list of all catalog database files. NOTE that .cat files are either placed in database_path directly, or by scbuild()
        dblist = glob.glob( gconfig.database_path+'*.'+gconfig.database_ext )
        # Load the catalog file(s)
        catalog = []
        for db in dblist:
            with open( db , 'rb') as dbf:
                catalog = catalog + pickle.load( dbf )

    # mass-ratio
    if q is not None:
        # handle int of float input
        if isinstance(q,(int,float)): q = [q-1e-6,q+1e-6]
        # NOTE: this could use error checking
        test = lambda k: k.m1/k.m2 >= min(q) and k.m1/k.m2 <= max(q)
        catalog = filter( test, catalog )

    # nonspinning
    if nonspinning is True:
        test = lambda k: norm(k.S1)+norm(k.S2) < tol
        catalog = filter( test, catalog )

    # spin aligned with orbital angular momentum
    if spinaligned is True:
        test = lambda k: allclose( dot(k.S1,k.L1+k.L2) , norm(k.S1)*norm(k.L1+k.L2) , atol=tol ) and allclose( dot(k.S2,k.L1+k.L2) , norm(k.S2)*norm(k.L1+k.L2) , atol=tol ) and not allclose( norm(k.S1)+norm(k.S2), 0.0, atol=tol )
        catalog = filter( test, catalog )

    # spin anti-aligned with orbital angular momentum
    if spinantialigned is True:
        test = lambda k: allclose( dot(k.S1,k.L1+k.L2) , -norm(k.S1)*norm(k.L1+k.L2) , atol=tol ) and allclose( dot(k.S2,k.L1+k.L2) , -norm(k.S2)*norm(k.L1+k.L2) , atol=tol ) and not allclose( norm(k.S1)+norm(k.S2), 0.0, atol=tol )
        catalog = filter( test, catalog )

    # precessing
    if precessing is True:
        test = lambda k: not allclose( abs(dot(k.S1+k.S2,k.L1+k.L2)), norm(k.L1+k.L2)*norm(k.S1+k.S2) , atol = tol )
        catalog = filter( test, catalog )

    # not precessing, same as spinaligned & spin anti aligned
    if notprecessing is True:
        test = lambda k: allclose( abs(dot(k.S1+k.S2,k.L1+k.L2))/(norm(k.L1+k.L2)*norm(k.S1+k.S2)), 1.0 , atol = tol )
        catalog = filter( test, catalog )

    # spins have equal magnitude
    if equalspin is True:
        test = lambda k: allclose( norm(k.S1), norm(k.S2), atol = tol )
        catalog = filter( test, catalog )

    # spins have unequal magnitude
    if unequalspin is True:
        test = lambda k: not allclose( norm(k.S1), norm(k.S2), atol = tol )
        catalog = filter( test, catalog )

    #
    if antialigned is True:
        test = lambda k: allclose( dot(k.S1+k.S2,k.L1+k.L2)/(norm(k.S1+k.S2)*norm(k.L1+k.L2)), -1.0, atol = tol )
        catalog = filter( test, catalog )

    # Compare setname strings
    if setname is not None:
        if isinstance( setname, str ):
            setname = [setname]
        setname = filter( lambda s: isinstance(s,str), setname )
        setname = [ k.lower() for k in setname ]
        if isinstance( setname, list ) and len(setname)>0:
            test = lambda k: k.setname.lower() in setname
            catalog = filter( test, catalog )
        else:
            msg = '[%s]>> setname input must be nonempty string or list.' % thisfun
            raise ValueError(msg)

    # Compare not setname strings
    if notsetname is not None:
        if isinstance( notsetname, str ):
            notsetname = [notsetname]
        notsetname = filter( lambda s: isinstance(s,str), notsetname )
        notsetname = [ k.lower() for k in notsetname ]
        if isinstance( notsetname, list ) and len(notsetname)>0:
            test = lambda k: not ( k.setname.lower() in notsetname )
            catalog = filter( test, catalog )
        else:
            msg = '[%s]>> notsetname input must be nonempty string or list.' % thisfun
            raise ValueError(msg)

    # Compare institute strings
    if institute is not None:
        if isinstance( institute, str ):
            institute = [institute]
        institute = filter( lambda s: isinstance(s,str), institute )
        institute = [ k.lower() for k in institute ]
        if isinstance( institute, list ) and len(institute)>0:
            test = lambda k: k.config.institute.lower() in institute
            catalog = filter( test, catalog )
        else:
            msg = '[%s]>> institute input must be nonempty string or list.' % thisfun
            raise ValueError(msg)

    # Compare keyword
    if keyword is not None:
        if isinstance( keyword, str ):
            keyword = [keyword]
        keyword = filter( lambda s: isinstance(s,str), keyword )
        keyword = [ k.lower() for k in keyword ]
        for w in keyword:
            test = lambda k: w in k.metadata_file_location.lower()
            catalog = filter( test, catalog )

    # Compare not keyword
    if notkeyword is not None:
        if isinstance( notkeyword, str ):
            notkeyword = [notkeyword]
        notkeyword = filter( lambda s: isinstance(s,str), notkeyword )
        notkeyword = [ k.lower() for k in notkeyword ]
        for w in notkeyword:
            test = lambda k: not ( w in k.metadata_file_location.lower() )
            catalog = filter( test, catalog )

    # Filter out physically degenerate simuations within a default tolerance
    output_descriptor = magenta(' possibly degenerate')
    if unique:
        catalog = scunique(catalog,verbose=False)
        output_descriptor = green(' unique')

    # Sort by final dimnesionless spin
    catalog = sorted( catalog, key = lambda e: e.xf, reverse = True )

    #
    if verbose:
        if len(catalog)>0:
            print '## Found %s%s simulations:' % ( bold(str(len(catalog))), output_descriptor )
            for k,entry in enumerate(catalog):
                print '[%04i] %s: %s' % ( k+1, cyan(entry.setname), entry.label )
        else:
            print red('!! Found %s simulations.' % str(len(catalog)))
        print ''

    #
    return catalog<commit_msg>now scsearch with a list of keywords will return the set union of results<commit_after>
'''
Modules for Numerical Relativity Simulation Catalog:

* catalog: builds catalog given a cinfiguration file, or directory containing many configuration files.

* scentry: class for simulation catalog entry (should include io)

'''

#
from nrutils.core import settings as gconfig
from nrutils.core.basics import *
from nrutils.core import M_RELATIVE_SIGN_CONVENTION
import warnings,sys

# Class representation of configuration files. The contents of these files define where the metadata for each simulation is stored, and where the related NR data is stored.
class scconfig(smart_object):

    # Create scconfig object from configuration file location
    def __init__(this,config_file_location=None):

        # Required fields from smart_object
        this.source_file_path = []
        this.source_dir = []

        # call wrapper for constructor
        this.config_file_location = config_file_location
        this.reconfig()

    # The actual constructor: this will be called within utility functions so that scentry objects are configured with local settings.
    def reconfig(this):

        #
        if this.config_file_location is None:
            msg = '(!!) scconfig objects cannot be initialted/reconfigured without a defined "config_file_location" location property (i.e. string where the related config file lives)'
            raise ValueError(msg)

        # learn the contents of the configuration file
        this.learn_file( this.config_file_location, comment=[';','#'] )

        # validate the information learned from the configuration file against minimal standards
        this.valid = this.validate()

        # In some cases, it is useful to have this function return this
        return this

    # Validate the config file against a minimal set of required fields.
    def validate(this):

        # Import useful things
        from os.path import expanduser

        # Create a string with the current process name
        thisfun = inspect.stack()[0][3]

        # each scconfig object (and the realted file) MUST have the following attributes
        required_attrs = [ 'institute',                 # school or collaboration authoring run
                           'metadata_id',               # unique string that defines metadata files
                           'catalog_dir',               # local directory where all simulation folders are stored
                                                        # this directory allows catalog files to be portable
                           'data_file_name_format',     # formatting string for referencing l m and extraction parameter
                           'handler_location',          # location of python script which contains validator and
                                                        # learn_metadata functions
                           'is_extrapolated',           # users should set this to true if waveform is extrapolated
                                                        # to infinity
                           'default_par_list' ]   # list of default parameters for loading: default_extraction_parameter, default_level. NOTE that list must be of length 2

        # Make sure that each required attribute is a member of this objects dictionary representation. If it's not, throw an error.
        for attr in required_attrs:
            if not ( attr in this.__dict__ ):
                msg = '(!!) Error -- config file at %s does NOT contain required field %s' % (  magenta(this.config_file_location), attr )
                raise ValueError(msg)

        # Make sure that data_file_name_format is list of strings. The intention is to give the user the ability to define multiple formats for loading. For example, the GT dataset may have files that begin with Ylm_Weyl... and others that begin with mp_Weylscalar... .
        if isinstance( this.data_file_name_format, str ):
            this.data_file_name_format = [this.data_file_name_format]
        elif isinstance(this.data_file_name_format,list):
            for k in this.data_file_name_format:
                if not isinstance(k,str):
                    msg = '(!!) Error in %s: each element of data_file_name_format must be character not numeric. Found data_file_name_format = %s' % (magenta(this.config_file_location),k)
                    raise ValueError(msg)
                if len( k.split('%i') ) != 4:
                    msg = '(!!) Error in %s: All elements of data_file_name_format must have three integer formatting tags (%%i). The offending entry is %s.' % ( magenta(this.config_file_location), red(k) )
                    raise ValueError(msg)
        else:
            msg = '(!!)  Error in %s: data_file_name_format must be comma separated list.' %  magenta(this.config_file_location)

        # Make sure that catalog_dir is string
        if not isinstance( this.catalog_dir, str ):
            msg = 'catalog_dir values must be string'
            error(red(msg),thisfun)


        if 2 != len(this.default_par_list):
            msg = '(!!) Error in %s: default_par_list must be list containing default extraction parameter (Numeric value) and default level (also Numeric in value). Invalide case found: %s' % (magenta(this.config_file_location),list(this.default_par_list))
            raise ValueError(msg)

        # Make sure that all directories end with a forward slash
        for attr in this.__dict__:
            if 'dir' in attr:
                if this.__dict__[attr][-1] != '/':
                    this.__dict__[attr] += '/'
        # Make sure that user symbols (~) are expanded
        for attr in this.__dict__:
            if ('dir' in attr) or ('location' in attr):
                if isinstance(this.__dict__[attr],str):
                    this.__dict__[attr] = expanduser( this.__dict__[attr] )
                elif isinstance(this.__dict__[attr],list):
                    for k in this.__dict__[attr]:
                        if isinstance(k,str):
                            k = expanduser(k)

# Class for simulation catalog e.
class scentry:

    # Create scentry object given location of metadata file
    def __init__( this, config_obj, metadata_file_location ):

        # Keep an internal log for each scentry created
        this.log = '[Log for %s] The file is "%s".' % (this,metadata_file_location)

        # Store primary inputs as object attributes
        this.config = config_obj
        this.metadata_file_location = metadata_file_location

        # Validate the location of the metadata file: does it contain waveform information? is the file empty? etc
        this.is_valid = this.validate()

        # If valid, learn metadata. Note that metadata property are defined as none otherise. Also NOTE that the standard metadata is stored directly to this object's attributes.
        this.raw_metadata = None
        if this.is_valid is True:
            #
            print '## Working: %s' % cyan(metadata_file_location)
            this.log += ' This entry\'s metadata file is valid.'

            # i.e. learn the meta_data_file
            try:
                this.learn_metadata()
                this.label = sclabel( this )
            except:
                emsg = sys.exc_info()[1].message
                this.log += '%80s'%' [FATALERROR-1] The metadata failed to be read. There may be an external formatting inconsistency. It is being marked as invalid with None. The system says: %s'%emsg
                warning( 'The following error message will be logged: '+red(emsg),'scentry')
                this.is_valid = None # An external program may use this to do something
                this.label = 'invalid!'

        elif this.is_valid is False:
            print '## The following is '+red('invalid')+': %s' % cyan(metadata_file_location)
            this.log += ' This entry\'s metadta file is invalid.'

    #
    def validate(this):

        # determine the validity of the current object

        # import validation function given in config file
        from imp import load_source
        validator_module = load_source( '', this.config.handler_location )

        # Name the function representation that will be used to load the metadata file, and convert it to raw and standardized metadata
        validator = validator_module.validate

        # vet the directory where the metadata file lives for: waveform and additional metadata
        status = validator( this.metadata_file_location, config = this.config )

        #
        return status

    # Standardize metadata
    def learn_metadata(this):

        #
        from imp import load_source
        from numpy import allclose

        # import metadata converter functiongiven in config file
        converter_module = load_source( '', this.config.handler_location )

        # Name the function representation that will be used to load the metadata file, and convert it to raw and standardized metadata
        learn_institute_metadata = converter_module.learn_metadata

        # Eval and store standard metadata
        [standard_metadata, this.raw_metadata] = learn_institute_metadata( this.metadata_file_location )

        # Validate the standard metadata
        required_attrs = [ 'date_number',   # creation date (number!) of metadata file
                           'note',          # informational note relating to metadata
                           'madm',          # initial ADM mass = m1+m2 - initial binding energy
                           'b',             # initial orbital separation (scalar: M)
                           'R1', 'R2',      # initial component masses (scalars: M = m1+m2)
                           'm1', 'm2',      # initial component masses (scalars: M = m1+m2)
                           'P1', 'P2',      # initial component linear momenta (Vectors ~ M )
                           'L1', 'L2',      # initial component angular momental (Vectors ~ M)
                           'S1', 'S2',      # initial component spins (Vectors ~ M*M)
                           'mf', 'Sf',      # Final mass (~M) and final dimensionful spin (~M*M)
                           'extraction_parameter_is_radius', # If False, then raw data must be scaled by the extraction
                                                             # radius (it's ~ 1 ); else, raw data is not scaled by the
                                                             # extractino radius (it's ~ 1/r, and it will need to be
                                                             # scaled by r to then be proportional to 1).
                           'xf' ]           # Final dimensionless spin *Magnitude*: xf = |Sf|/(mf*mf)

        for attr in required_attrs:
            if attr not in standard_metadata.__dict__:
                msg = '(!!) Error -- Output of %s does NOT contain required field %s' % ( this.config.handler_location, attr )
                raise ValueError(msg)

        # Confer the required attributes to this object for ease of referencing
        for attr in standard_metadata.__dict__.keys():
            setattr( this, attr, standard_metadata.__dict__[attr] )

        # tag this entry with its inferred setname
        this.setname = this.raw_metadata.source_dir[-1].split( this.config.catalog_dir )[-1].split('/')[0]

        # tag this entry with the directory location of the metadata file. NOTE that the waveform data must be reference relative to this directory via config.data_file_name_format
        this.relative_simdir = this.raw_metadata.source_dir[-1].split( this.config.catalog_dir )[-1]

        # Basic sanity check for standard attributes. NOTE this section needs to be completed and perhaps externalized to the current function.

        # Check that initial binary separation is float
        if not isinstance( this.b , float ) :
            msg = 'b = %g' % this.b
            raise ValueError(msg)
        # Check that final mass is float
        if not isinstance( this.mf , float ) :
            msg = ''
            raise ValueError(msg)
        # Check that inital mass1 is float
        if not isinstance( this.m1 , float ) :
            msg = ''
            raise ValueError(msg)
        # Check that inital mass2 is float
        if not isinstance( this.m2 , float ) :
            msg = ''
            raise ValueError(msg)
        # Enfore m1>m2 convention.
        satisfies_massratio_convetion = lambda e: (not e.m1 > e.m2) and (not allclose(e.m1,e.m2,atol=1e-4))
        if satisfies_massratio_convetion(this):
            this.flip()
        if satisfies_massratio_convetion(this):
            msg = 'Mass ratio convention m1>m2 must be used. Check scentry.flip(). It should have corrected this! \n>> m1 = %g, m2 = %g' % (this.m1,this.m2)
            raise ValueError(msg)

    # Create dynamic function that references the user's current configuration to construct the simulation directory of this run.
    def simdir(this): return this.config.reconfig().catalog_dir + this.relative_simdir

    # Flip 1->2 associations.
    def flip(this):

        #
        from numpy import array,double

        # Store the flippoed variables to placeholders
        R1 = array(this.R2); R2 = array(this.R1);
        m1 = double(this.m2); m2 = double(this.m1);
        P1 = array(this.P2); P2 = array(this.P1);
        L1 = array(this.L2); L2 = array(this.L1);
        S1 = array(this.S2); S2 = array(this.S1);

        # Apply the flip to the current object
        this.R1 = R1; this.R2 = R2
        this.m1 = m1; this.m2 = m2
        this.P1 = P1; this.P2 = P2
        this.L1 = L1; this.L2 = L2
        this.S1 = S1; this.S2 = S2

    # Compare this scentry object to another using initial parameter fields. Return true false statement
    def compare2( this, that, atol=1e-3 ):

        #
        from numpy import allclose,hstack,double

        # Calculate an array of initial parameter values (the first element is 0 or 1 describing quasi-circularity)
        def param_array( entry ):

            # List of fields to add to array: initial parameters that are independent of initial separation
            field_list = [ 'm1', 'm2', 'S1', 'S2' ]

            #
            a = double( 'qc' in entry.label )
            for f in field_list:
                a = hstack( [a, entry.__dict__[f] ] )

            #
            return a

        # Perform comparison and return
        return allclose( param_array(this), param_array(that), atol=atol )



# Create the catalog database, and store it as a pickled file.
def scbuild(save=True):

    # Load useful packages
    from commands import getstatusoutput as bash
    from os.path import realpath, abspath, join, splitext, basename
    from os import pardir,system,popen
    import pickle

    # Create a string with the current process name
    thisfun = inspect.stack()[0][3]

    # Look for config files
    cpath_list = glob.glob( gconfig.config_path+'*.ini' )

    #
    if not cpath_list:
        msg = 'Cannot find configuration files (*.ini) in %s' % gconfig.config_path
        error(msg,thisfun)

    # Create config objects from list of config files
    configs = [ scconfig( config_path ) for config_path in cpath_list ]

    # For earch config
    for config in configs:

        # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #
        # Create streaming log file        #
        logfstr = gconfig.database_path + '/' + splitext(basename(config.config_file_location))[0] + '.log'
        msg = 'Opening log file in: '+cyan(logfstr)
        alert(msg,thisfun)
        logfid = open(logfstr, 'w')
        # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #

        # Search recurssively within the config's catalog_dir for files matching the config's metadata_id
        msg = 'Searching for %s in %s.' % ( cyan(config.metadata_id), cyan(config.catalog_dir) ) + yellow(' This may take a long time if the folder being searched is mounted from a remote drive.')
        alert(msg,thisfun)
        mdfile_list = rfind(config.catalog_dir,config.metadata_id,verbose=True)
        alert('done.',thisfun)

        # (try to) Create a catalog entry for each valid metadata file
        config.catalog = []
        h = -1
        for mdfile in mdfile_list:

            # Create tempoary scentry object
            entry = scentry(config,mdfile)

            # Write to the master log file
            h+=1
            logfid.write( '%5i\t%s\n'% (h,entry.log) )

            # If the obj is valid, add it to the catalog list, else ignore
            if entry.is_valid:
                config.catalog.append( entry )
            else:
                del entry

        # Store the catalog to the database_path
        if save:
            db = gconfig.database_path + '/' + splitext(basename(config.config_file_location))[0] + '.' + gconfig.database_ext
            with open(db, 'wb') as dbf:
                pickle.dump( config.catalog , dbf, pickle.HIGHEST_PROTOCOL )

        # Close the log file
        logfid.close()




# Function for searching through catalog files.
def scsearch( catalog = None,           # Manually input list of scentry objects to search through
              q = None,                 # RANGE of mass ratios (>=1) to search for
              nonspinning = None,       # Non-spinning initially
              spinaligned = None,       # spin-aligned with L AND no in-plane spin INITIALLY
              spinantialigned = None,   # spin-anti-aligned with L AND no in-plane spin INITIALLY
              precessing = None,        # not spin aligned
              notprecessing = None,     # not precessing
              equalspin = None,         # equal spin magnitudes
              unequalspin = None,       # not equal spin magnitudes
              antialigned = None,       # spin is in opposite direction of L
              setname = None,           # name of simulation set
              notsetname = None,        # list of setnames to ignore
              institute = None,         # list of institutes to accept
              keyword = None,           # list of keywords to accept (based on metadata directory string)
              notkeyword = None,        # list of keywords to not accept (based on metadata
                                        # directory string
              unique = None,            # if true, only simulations with unique initial conditions will be used
              plot = None,              # whether or not to show a plot of results
              verbose = None):          # be verbose

    # Print non None inputs to screen
    thisfun = inspect.stack()[0][3]
    if verbose is not None:
        for k in dir():
            if (eval(k) is not None) and (k != 'thisfun'):
                print '[%s]>> Found %s (=%r) keyword.' % (thisfun,textul(k),eval(k))

    '''
    Handle individual cases:
    '''

    #
    from os.path import realpath, abspath, join
    from os import pardir
    from numpy.linalg import norm
    from numpy import allclose,dot
    import pickle, glob

    # absolute tolerance for num comparisons
    tol = 1e-6

    # Handle the catalog input
    if catalog is None:
        # Get a list of all catalog database files. NOTE that .cat files are either placed in database_path directly, or by scbuild()
        dblist = glob.glob( gconfig.database_path+'*.'+gconfig.database_ext )
        # Load the catalog file(s)
        catalog = []
        for db in dblist:
            with open( db , 'rb') as dbf:
                catalog = catalog + pickle.load( dbf )

    # mass-ratio
    if q is not None:
        # handle int of float input
        if isinstance(q,(int,float)): q = [q-1e-6,q+1e-6]
        # NOTE: this could use error checking
        test = lambda k: k.m1/k.m2 >= min(q) and k.m1/k.m2 <= max(q)
        catalog = filter( test, catalog )

    # nonspinning
    if nonspinning is True:
        test = lambda k: norm(k.S1)+norm(k.S2) < tol
        catalog = filter( test, catalog )

    # spin aligned with orbital angular momentum
    if spinaligned is True:
        test = lambda k: allclose( dot(k.S1,k.L1+k.L2) , norm(k.S1)*norm(k.L1+k.L2) , atol=tol ) and allclose( dot(k.S2,k.L1+k.L2) , norm(k.S2)*norm(k.L1+k.L2) , atol=tol ) and not allclose( norm(k.S1)+norm(k.S2), 0.0, atol=tol )
        catalog = filter( test, catalog )

    # spin anti-aligned with orbital angular momentum
    if spinantialigned is True:
        test = lambda k: allclose( dot(k.S1,k.L1+k.L2) , -norm(k.S1)*norm(k.L1+k.L2) , atol=tol ) and allclose( dot(k.S2,k.L1+k.L2) , -norm(k.S2)*norm(k.L1+k.L2) , atol=tol ) and not allclose( norm(k.S1)+norm(k.S2), 0.0, atol=tol )
        catalog = filter( test, catalog )

    # precessing
    if precessing is True:
        test = lambda k: not allclose( abs(dot(k.S1+k.S2,k.L1+k.L2)), norm(k.L1+k.L2)*norm(k.S1+k.S2) , atol = tol )
        catalog = filter( test, catalog )

    # not precessing, same as spinaligned & spin anti aligned
    if notprecessing is True:
        test = lambda k: allclose( abs(dot(k.S1+k.S2,k.L1+k.L2))/(norm(k.L1+k.L2)*norm(k.S1+k.S2)), 1.0 , atol = tol )
        catalog = filter( test, catalog )

    # spins have equal magnitude
    if equalspin is True:
        test = lambda k: allclose( norm(k.S1), norm(k.S2), atol = tol )
        catalog = filter( test, catalog )

    # spins have unequal magnitude
    if unequalspin is True:
        test = lambda k: not allclose( norm(k.S1), norm(k.S2), atol = tol )
        catalog = filter( test, catalog )

    #
    if antialigned is True:
        test = lambda k: allclose( dot(k.S1+k.S2,k.L1+k.L2)/(norm(k.S1+k.S2)*norm(k.L1+k.L2)), -1.0, atol = tol )
        catalog = filter( test, catalog )

    # Compare setname strings
    if setname is not None:
        if isinstance( setname, str ):
            setname = [setname]
        setname = filter( lambda s: isinstance(s,str), setname )
        setname = [ k.lower() for k in setname ]
        if isinstance( setname, list ) and len(setname)>0:
            test = lambda k: k.setname.lower() in setname
            catalog = filter( test, catalog )
        else:
            msg = '[%s]>> setname input must be nonempty string or list.' % thisfun
            raise ValueError(msg)

    # Compare not setname strings
    if notsetname is not None:
        if isinstance( notsetname, str ):
            notsetname = [notsetname]
        notsetname = filter( lambda s: isinstance(s,str), notsetname )
        notsetname = [ k.lower() for k in notsetname ]
        if isinstance( notsetname, list ) and len(notsetname)>0:
            test = lambda k: not ( k.setname.lower() in notsetname )
            catalog = filter( test, catalog )
        else:
            msg = '[%s]>> notsetname input must be nonempty string or list.' % thisfun
            raise ValueError(msg)

    # Compare institute strings
    if institute is not None:
        if isinstance( institute, str ):
            institute = [institute]
        institute = filter( lambda s: isinstance(s,str), institute )
        institute = [ k.lower() for k in institute ]
        if isinstance( institute, list ) and len(institute)>0:
            test = lambda k: k.config.institute.lower() in institute
            catalog = filter( test, catalog )
        else:
            msg = '[%s]>> institute input must be nonempty string or list.' % thisfun
            raise ValueError(msg)

    # Compare keyword
    if keyword is not None:
        if isinstance( keyword, str ):
            keyword = [keyword]
        keyword = filter( lambda s: isinstance(s,str), keyword )
        keyword = [ k.lower() for k in keyword ]
        temp_catalogs = [ catalog for w in keyword ]
        new_catalog = []
        for j,key in enumerate(keyword):
            test = lambda k: key in k.metadata_file_location.lower()
            new_catalog += filter( test, temp_catalogs[j] )
        catalog = list(set(new_catalog))

    # Compare not keyword
    if notkeyword is not None:
        if isinstance( notkeyword, str ):
            notkeyword = [notkeyword]
        notkeyword = filter( lambda s: isinstance(s,str), notkeyword )
        notkeyword = [ k.lower() for k in notkeyword ]
        for w in notkeyword:
            test = lambda k: not ( w in k.metadata_file_location.lower() )
            catalog = filter( test, catalog )

    # Filter out physically degenerate simuations within a default tolerance
    output_descriptor = magenta(' possibly degenerate')
    if unique:
        catalog = scunique(catalog,verbose=False)
        output_descriptor = green(' unique')

    # Sort by final dimnesionless spin
    catalog = sorted( catalog, key = lambda e: e.date_number, reverse = True )

    #
    if verbose:
        if len(catalog)>0:
            print '## Found %s%s simulations:' % ( bold(str(len(catalog))), output_descriptor )
            for k,entry in enumerate(catalog):
                print '[%04i] %s: %s' % ( k+1, cyan(entry.setname), entry.label )
        else:
            print red('!! Found %s simulations.' % str(len(catalog)))
        print ''

    #
    return catalog
{'fraction_non_alphanumeric': 0.0889568712134845, 'fraction_numerical': 0.00783796212984624, 'mean_word_length': 2.097033872066382, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4768096', 'n_tokens_mistral': 13865, 'n_tokens_neox': 13441, 'n_words': 4827}