<commit_before>#
# Project:
#   glideinWMS
#
# File Version: 
#
# Desscription:
#   This module contains the generic params classes
#
# Extracted from:
#   cgWParams.py
#
# Author:
#   <NAME>
#

import os
import copy
import sys
import os.path
import string
import socket
import types
import traceback
import xmlParse
import xml.parsers.expat
import xmlFormat

class SubParams:
    def __init__(self,data):
        self.data=data

    def __eq__(self,other):
        if other==None:
            return False
        if not isinstance(other,self.__class__):
            return False
        return self.data==other.data

    # make data elements look like class attributes
    def __getattr__(self,name):
        return self.get_el(name)

    # make data elements look like a dictionary
    def keys(self):
        return self.data.keys()
    def has_key(self,name):
        return self.data.has_key(name)
    def __getitem__(self,name):
        return self.get_el(name)    
    def __repr__(self):
        return str(self.data)
    def __str__(self):
        return str(self.data)

    #
    # PROTECTED
    #

    # validate input against bae template (i.e. the defaults)
    def validate(self,base,path_text):
        for k in self.data.keys():
            if not base.has_key(k):
                # element not in base, report
                raise RuntimeError, "Unknown parameter %s.%s"%(path_text,k)
            else:
                # verify subelements, if any
                defel=base[k]
                if isinstance(defel,xmlParse.OrderedDict):
                    # subdictionary
                    self[k].validate(defel,"%s.%s"%(path_text,k))
                else:
                    # final element
                    defvalue,ktype,txt,subdef=defel

                    if isinstance(defvalue,xmlParse.OrderedDict):
                        # dictionary el elements
                        data_el=self[k]
                        for data_subkey in data_el.keys():
                            data_el[data_subkey].validate(subdef,"%s.%s.%s"%(path_text,k,data_subkey))
                    elif type(defvalue)==type([]):
                        # list of elements
                        if isinstance(self.data[k],xmlParse.OrderedDict):
                            if len(self.data[k].keys())==0:
                                self.data[k]=[]  #XML does not know if an empty list is a dictionary or not.. fix this

                        mylist=self[k]
                        if type(mylist)!=type([]):
                            raise RuntimeError, "Parameter %s.%s not a list: %s %s"%(path_text,k,type(mylist),mylist)
                        for data_el in mylist:
                            data_el.validate(subdef,"%s.*.%s"%(path_text,k))
                    else:
                        # a simple value
                        pass #nothing to be done

    # put default values where there is nothing
    def use_defaults(self,defaults):
        for k in defaults.keys():
            defel=defaults[k]
            if isinstance(defel,xmlParse.OrderedDict):
                # subdictionary
                if not self.data.has_key(k):
                    self.data[k]=xmlParse.OrderedDict() # first create empty, if does not exist

                # then, set defaults on all elements of subdictionary
                self[k].use_defaults(defel)
            else:
                # final element
                defvalue,ktype,txt,subdef=defel

                if isinstance(defvalue,xmlParse.OrderedDict):
                    # dictionary el elements
                    if not self.data.has_key(k):
                        self.data[k]=xmlParse.OrderedDict() # no elements yet, set and empty dictionary
                    else:
                        # need to set defaults on all elements in the dictionary
                        data_el=self[k]
                        for data_subkey in data_el.keys():
                            data_el[data_subkey].use_defaults(subdef)
                elif type(defvalue)==type([]):
                    # list of elements
                    if not self.data.has_key(k):
                        self.data[k]=[] # no elements yet, set and empty list
                    else:
                        # need to set defaults on all elements in the list
                        for data_el in self[k]:
                            data_el.use_defaults(subdef)
                else:
                    # a simple value
                    if not self.data.has_key(k):
                        self.data[k]=copy.deepcopy(defvalue)
                    # else nothing to do, already set

            

    #
    # PRIVATE
    #
    def get_el(self,name):
        el=self.data[name]
        if isinstance(el,xmlParse.OrderedDict):
            return self.__class__(el)
        elif type(el)==type([]):
            outlst=[]
            for k in el:
                if isinstance(k,xmlParse.OrderedDict):
                    outlst.append(self.__class__(k))
                else:
                    outlst.append(k)
            return outlst
        else:
            return el

# abstract class
# children must define
#   get_top_element(self)
#   init_defaults(self)
#   derive(self)
#   get_xml_format(self)
class Params:
    def __init__(self,usage_prefix,src_dir,argv):
        self.usage_prefix=usage_prefix

        # support dir
        self.src_dir=src_dir

        # initialize the defaults
        self.defaults=xmlParse.OrderedDict()
        self.init_defaults()

        try:
            if len(argv)<2:
                raise RuntimeError, "Missing config file"

            if argv[1]=="-help":
                raise RuntimeError,"\nA config file will contain:\n%s\n\nThe config file will be in XML format."%self.get_description("  ")
                
            self.cfg_name=argv[1]
            self.load_file(self.cfg_name)

            self.subparams.validate(self.defaults,self.get_top_element())

            # make a copy of the loaded data, so that I can always tell what was derived and what was not
            self.org_data=copy.deepcopy(self.data)

            self.subparams.use_defaults(self.defaults)
            
            # create derived values<commit_msg>Bug 2808: Using abspath for config name<commit_after>#
# Project:
#   glideinWMS
#
# File Version: 
#
# Desscription:
#   This module contains the generic params classes
#
# Extracted from:
#   cgWParams.py
#
# Author:
#   <NAME>
#

import os
import copy
import sys
import os.path
import string
import socket
import types
import traceback
import xmlParse
import xml.parsers.expat
import xmlFormat

class SubParams:
    def __init__(self,data):
        self.data=data

    def __eq__(self,other):
        if other==None:
            return False
        if not isinstance(other,self.__class__):
            return False
        return self.data==other.data

    # make data elements look like class attributes
    def __getattr__(self,name):
        return self.get_el(name)

    # make data elements look like a dictionary
    def keys(self):
        return self.data.keys()
    def has_key(self,name):
        return self.data.has_key(name)
    def __getitem__(self,name):
        return self.get_el(name)    
    def __repr__(self):
        return str(self.data)
    def __str__(self):
        return str(self.data)

    #
    # PROTECTED
    #

    # validate input against bae template (i.e. the defaults)
    def validate(self,base,path_text):
        for k in self.data.keys():
            if not base.has_key(k):
                # element not in base, report
                raise RuntimeError, "Unknown parameter %s.%s"%(path_text,k)
            else:
                # verify subelements, if any
                defel=base[k]
                if isinstance(defel,xmlParse.OrderedDict):
                    # subdictionary
                    self[k].validate(defel,"%s.%s"%(path_text,k))
                else:
                    # final element
                    defvalue,ktype,txt,subdef=defel

                    if isinstance(defvalue,xmlParse.OrderedDict):
                        # dictionary el elements
                        data_el=self[k]
                        for data_subkey in data_el.keys():
                            data_el[data_subkey].validate(subdef,"%s.%s.%s"%(path_text,k,data_subkey))
                    elif type(defvalue)==type([]):
                        # list of elements
                        if isinstance(self.data[k],xmlParse.OrderedDict):
                            if len(self.data[k].keys())==0:
                                self.data[k]=[]  #XML does not know if an empty list is a dictionary or not.. fix this

                        mylist=self[k]
                        if type(mylist)!=type([]):
                            raise RuntimeError, "Parameter %s.%s not a list: %s %s"%(path_text,k,type(mylist),mylist)
                        for data_el in mylist:
                            data_el.validate(subdef,"%s.*.%s"%(path_text,k))
                    else:
                        # a simple value
                        pass #nothing to be done

    # put default values where there is nothing
    def use_defaults(self,defaults):
        for k in defaults.keys():
            defel=defaults[k]
            if isinstance(defel,xmlParse.OrderedDict):
                # subdictionary
                if not self.data.has_key(k):
                    self.data[k]=xmlParse.OrderedDict() # first create empty, if does not exist

                # then, set defaults on all elements of subdictionary
                self[k].use_defaults(defel)
            else:
                # final element
                defvalue,ktype,txt,subdef=defel

                if isinstance(defvalue,xmlParse.OrderedDict):
                    # dictionary el elements
                    if not self.data.has_key(k):
                        self.data[k]=xmlParse.OrderedDict() # no elements yet, set and empty dictionary
                    else:
                        # need to set defaults on all elements in the dictionary
                        data_el=self[k]
                        for data_subkey in data_el.keys():
                            data_el[data_subkey].use_defaults(subdef)
                elif type(defvalue)==type([]):
                    # list of elements
                    if not self.data.has_key(k):
                        self.data[k]=[] # no elements yet, set and empty list
                    else:
                        # need to set defaults on all elements in the list
                        for data_el in self[k]:
                            data_el.use_defaults(subdef)
                else:
                    # a simple value
                    if not self.data.has_key(k):
                        self.data[k]=copy.deepcopy(defvalue)
                    # else nothing to do, already set

            

    #
    # PRIVATE
    #
    def get_el(self,name):
        el=self.data[name]
        if isinstance(el,xmlParse.OrderedDict):
            return self.__class__(el)
        elif type(el)==type([]):
            outlst=[]
            for k in el:
                if isinstance(k,xmlParse.OrderedDict):
                    outlst.append(self.__class__(k))
                else:
                    outlst.append(k)
            return outlst
        else:
            return el

# abstract class
# children must define
#   get_top_element(self)
#   init_defaults(self)
#   derive(self)
#   get_xml_format(self)
class Params:
    def __init__(self,usage_prefix,src_dir,argv):
        self.usage_prefix=usage_prefix

        # support dir
        self.src_dir=src_dir

        # initialize the defaults
        self.defaults=xmlParse.OrderedDict()
        self.init_defaults()

        try:
            if len(argv)<2:
                raise RuntimeError, "Missing config file"

            if argv[1]=="-help":
                raise RuntimeError,"\nA config file will contain:\n%s\n\nThe config file will be in XML format."%self.get_description("  ")
                
            self.cfg_name=os.path.abspath(argv[1])
            self.load_file(self.cfg_name)

            self.subparams.validate(self.defaults,self.get_top_element())

            # make a copy of the loaded data, so that I can always tell what was derived and what was not
            self.org_data=copy.deepcopy(self.data)

            self.subparams.use_defaults(self.defaults)
            
            # create derived values
{'fraction_non_alphanumeric': 0.0881860909742691, 'fraction_numerical': 0.0009559467856289333, 'mean_word_length': 1.5123073844306585, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3591720', 'n_tokens_mistral': 3495, 'n_tokens_neox': 3320, 'n_words': 841}