parameters: (self)
docstring: Fetch internal and external dataset configurations.
docstring_summary: Fetch internal and external dataset configurations.
docstring_tokens: Fetch internal and external dataset configurations .
function: def _fetch_datasets(self):
          """Fetch internal and external dataset configurations."""
  
          all_configs = collections.OrderedDict()
          all_configs["main"] = {"config": self._config,
                                 "general": self._general_options,
                                 "weight": None}
          all_configs.update(self._external_datasets)
  
          for config in all_configs.values():
              global_weight = "1" if config["weight"] is None else config["weight"]
              if global_weight+self._config.config_path in self._common_weights:
                  global_weight = self._common_weights[global_weight+
                                                       self._config.config_path]
              elif global_weight in self._common_weights:
                  global_weight = self._common_weights[global_weight]
              else:
                  global_weight = (global_weight,)
              common_weights = {}
              weight_dict = {}
              for section in config["config"].sections():
                  cache_datasetdir = os.environ["datasetdir"]
                  if "general" in section:
                      if config["config"].has_option("general", "datasetdir"):
                          os.environ["datasetdir"] = config["config"].get("general", "datasetdir")
                  elif section == "weights":
                      for option in config["config"].options(section):
                          common_weights[option] \
                              = [x.strip() for x in
                                 config["config"].get(section, option).split(",")]
                  elif section.startswith("dataset:"):
                      print("-"*75)
                      # set name from section-name
                      name = section[8:]
                      if name in self._datasets:
                          print("WARNING: Duplicate definition of dataset '{}'".format(name))
                          print(" -> Using defintion in '{}':\n".format(config["config"].config_path))
                          print("    [{}]".format(section))
                          for k,v in config["config"].items(section):
                              print("   ", k, "=", v)
                          print()
                      self._datasets[name] = {}
  
                      # extract weight for the dataset
                      if config["config"].has_option(section, "weight"):
                          self._weight_dict[name] \
                              = [x.strip() for x in
                                 config["config"].get(section, "weight").split(",")]
                      else:
                          self._weight_dict[name] = ["1.0"]
                      self._weight_dict[name] = [global_w+"*"+w
                                                 for w in self._weight_dict[name]
                                                 for global_w in global_weight]
                      weight_dict[name] = self._weight_dict[name]
  
                      # extract essential variables
                      for var in ("inputFileList", "collection"):
                          try:
                              self._datasets[name][var] = config["config"].get(section, var)
                          except ConfigParser.NoOptionError:
                              print("No", var, "found in", section+". Please check ini-file.")
                              sys.exit(1)
  
                      # get globaltag and configTemplate. If none in section, try to get
                      # default from [general] section.
                      for var in ("configTemplate", "globaltag"):
                          try:
                              self._datasets[name][var] = config["config"].get(section, var)
                          except (ConfigParser.NoSectionError,ConfigParser.NoOptionError):
                              try:
                                  self._datasets[name][var] = config["general"][var]
                              except KeyError:
                                  try:
                                      self._datasets[name][var] \
                                          = all_configs["main"]["general"][var]
                                  except KeyError:
                                      print("No",var,"found in ["+section+"]", end=' ')
                                      print("and no default in [general] section.")
                                      sys.exit(1)
  
                      # extract non-essential options
                      if "ALCARECOTkAlCosmics" in self._datasets[name]["collection"]:
                          try:
                              self._datasets[name]["cosmicsZeroTesla"] \
                                  = config["config"].getboolean(section,"cosmicsZeroTesla")
                          except ConfigParser.NoOptionError:
                              print("No option cosmicsZeroTesla found in", section,"even though it is required for dataset type", self._datasets[name]["collection"], ". Please check ini-file.")
                              sys.exit(1)
                          try:
                              self._datasets[name]["cosmicsDecoMode"] \
                                  = config["config"].getboolean(section,"cosmicsDecoMode")
                          except ConfigParser.NoOptionError:
                              print("No option cosmicsDecoMode found in", section,"even though it is required for dataset type", self._datasets[name]["collection"], ".Please check ini-file.")
                              sys.exit(1)
  
                      self._datasets[name]["primaryWidth"] = -1.0
                      if config["config"].has_option(section,"primaryWidth"):
                          self._datasets[name]["primaryWidth"] \
                              = config["config"].getfloat(section,"primaryWidth")
  
                      self._datasets[name]["numberOfEvents"] = -1
                      if config["config"].has_option(section, "numberOfEvents"):
                          self._datasets[name]["numberOfEvents"] \
                              = config["config"].getint(section, "numberOfEvents")
  
                      self._datasets[name]["json"] = ""
                      try:
                          self._datasets[name]["json"] = config["config"].get(section,"json")
                      except ConfigParser.NoOptionError:
                          try:
                              self._datasets[name]["json"] = config["general"]["json"]
                          except KeyError:
                              try:
                                  self._datasets[name]["json"] \
                                      = all_configs["main"]["general"]["json"]
                              except KeyError:
                                  print("No json given in either [general] or", end=' ')
                                  print("["+section+"] sections.")
                                  print(" -> Proceeding without json-file.")
  
  
                      #replace ${datasetdir} and other variables, e.g. $CMSSW_BASE
                      for var in ("inputFileList", "json", "configTemplate"):
                          self._datasets[name][var] \
                              = os.path.expandvars(self._datasets[name][var])
  
  
                      # Get number of jobs from lines in inputfilelist
                      self._datasets[name]["njobs"] = 0
                      try:
                          with open(self._datasets[name]["inputFileList"], "r") as filelist:
                              for line in filelist:
                                  if "CastorPool" in line:
                                      continue
                                  # ignore empty lines
                                  if not line.strip()=="":
                                      self._datasets[name]["njobs"] += 1
                      except IOError:
                          print("Inputfilelist", self._datasets[name]["inputFileList"], end=' ')
                          print("does not exist.")
                          sys.exit(1)
                      if self._datasets[name]["njobs"] == 0:
                          print("Number of jobs is 0. There may be a problem with the inputfilelist:")
                          print(self._datasets[name]["inputFileList"])
                          sys.exit(1)
  
                      # Check if njobs gets overwritten in .ini-file
                      if config["config"].has_option(section, "njobs"):
                          if config["config"].getint(section, "njobs") <= self._datasets[name]["njobs"]:
                              self._datasets[name]["njobs"] = config["config"].getint(section, "njobs")
                          else:
                              print("'njobs' is bigger than the number of files for this", end=' ')
                              print("dataset:", self._datasets[name]["njobs"])
                              print("Using default.")
                      else:
                          print("No number of jobs specified. Using number of files in", end=' ')
                          print("inputfilelist as the number of jobs.")
  
              # check if local weights override global weights and resolve name clashes
              for weight_name, weight_values in common_weights.items():
                  for key, weight in weight_dict.items():
                      if any([weight_name in w for w in weight]):
                          self._common_weights[weight_name+config["config"].config_path] = weight_values
                          self._weight_dict[key] = [mps_tools.replace_factors(w,
                                                                              weight_name,
                                                                              weight_name+config["config"].config_path)
                                                    for w in weight]
                      else:
                          self._common_weights[weight_name] = weight_values
                          self._weight_dict[key] = weight
  
              os.environ["datasetdir"] = cache_datasetdir
  
          if len(self._datasets) == 0:
              print("No dataset section defined in '{0}'".format(
                  ", ".join([self._args.aligmentConfig]+self._external_datasets.keys())))
              print("At least one section '[dataset:<name>]' is required.")
              sys.exit(1)
  
          self._global_tag = self._datasets[name]["globaltag"]
function_tokens: def _fetch_datasets ( self ) : all_configs = collections . OrderedDict ( ) all_configs [ "main" ] = { "config" : self . _config , "general" : self . _general_options , "weight" : None } all_configs . update ( self . _external_datasets ) for config in all_configs . values ( ) : global_weight = "1" if config [ "weight" ] is None else config [ "weight" ] if global_weight + self . _config . config_path in self . _common_weights : global_weight = self . _common_weights [ global_weight + self . _config . config_path ] elif global_weight in self . _common_weights : global_weight = self . _common_weights [ global_weight ] else : global_weight = ( global_weight , ) common_weights = { } weight_dict = { } for section in config [ "config" ] . sections ( ) : cache_datasetdir = os . environ [ "datasetdir" ] if "general" in section : if config [ "config" ] . has_option ( "general" , "datasetdir" ) : os . environ [ "datasetdir" ] = config [ "config" ] . get ( "general" , "datasetdir" ) elif section == "weights" : for option in config [ "config" ] . options ( section ) : common_weights [ option ] = [ x . strip ( ) for x in config [ "config" ] . get ( section , option ) . split ( "," ) ] elif section . startswith ( "dataset:" ) : print ( "-" * 75 ) # set name from section-name name = section [ 8 : ] if name in self . _datasets : print ( "WARNING: Duplicate definition of dataset '{}'" . format ( name ) ) print ( " -> Using defintion in '{}':\n" . format ( config [ "config" ] . config_path ) ) print ( "    [{}]" . format ( section ) ) for k , v in config [ "config" ] . items ( section ) : print ( "   " , k , "=" , v ) print ( ) self . _datasets [ name ] = { } # extract weight for the dataset if config [ "config" ] . has_option ( section , "weight" ) : self . _weight_dict [ name ] = [ x . strip ( ) for x in config [ "config" ] . get ( section , "weight" ) . split ( "," ) ] else : self . _weight_dict [ name ] = [ "1.0" ] self . _weight_dict [ name ] = [ global_w + "*" + w for w in self . _weight_dict [ name ] for global_w in global_weight ] weight_dict [ name ] = self . _weight_dict [ name ] # extract essential variables for var in ( "inputFileList" , "collection" ) : try : self . _datasets [ name ] [ var ] = config [ "config" ] . get ( section , var ) except ConfigParser . NoOptionError : print ( "No" , var , "found in" , section + ". Please check ini-file." ) sys . exit ( 1 ) # get globaltag and configTemplate. If none in section, try to get # default from [general] section. for var in ( "configTemplate" , "globaltag" ) : try : self . _datasets [ name ] [ var ] = config [ "config" ] . get ( section , var ) except ( ConfigParser . NoSectionError , ConfigParser . NoOptionError ) : try : self . _datasets [ name ] [ var ] = config [ "general" ] [ var ] except KeyError : try : self . _datasets [ name ] [ var ] = all_configs [ "main" ] [ "general" ] [ var ] except KeyError : print ( "No" , var , "found in [" + section + "]" , end = ' ' ) print ( "and no default in [general] section." ) sys . exit ( 1 ) # extract non-essential options if "ALCARECOTkAlCosmics" in self . _datasets [ name ] [ "collection" ] : try : self . _datasets [ name ] [ "cosmicsZeroTesla" ] = config [ "config" ] . getboolean ( section , "cosmicsZeroTesla" ) except ConfigParser . NoOptionError : print ( "No option cosmicsZeroTesla found in" , section , "even though it is required for dataset type" , self . _datasets [ name ] [ "collection" ] , ". Please check ini-file." ) sys . exit ( 1 ) try : self . _datasets [ name ] [ "cosmicsDecoMode" ] = config [ "config" ] . getboolean ( section , "cosmicsDecoMode" ) except ConfigParser . NoOptionError : print ( "No option cosmicsDecoMode found in" , section , "even though it is required for dataset type" , self . _datasets [ name ] [ "collection" ] , ".Please check ini-file." ) sys . exit ( 1 ) self . _datasets [ name ] [ "primaryWidth" ] = - 1.0 if config [ "config" ] . has_option ( section , "primaryWidth" ) : self . _datasets [ name ] [ "primaryWidth" ] = config [ "config" ] . getfloat ( section , "primaryWidth" ) self . _datasets [ name ] [ "numberOfEvents" ] = - 1 if config [ "config" ] . has_option ( section , "numberOfEvents" ) : self . _datasets [ name ] [ "numberOfEvents" ] = config [ "config" ] . getint ( section , "numberOfEvents" ) self . _datasets [ name ] [ "json" ] = "" try : self . _datasets [ name ] [ "json" ] = config [ "config" ] . get ( section , "json" ) except ConfigParser . NoOptionError : try : self . _datasets [ name ] [ "json" ] = config [ "general" ] [ "json" ] except KeyError : try : self . _datasets [ name ] [ "json" ] = all_configs [ "main" ] [ "general" ] [ "json" ] except KeyError : print ( "No json given in either [general] or" , end = ' ' ) print ( "[" + section + "] sections." ) print ( " -> Proceeding without json-file." ) #replace ${datasetdir} and other variables, e.g. $CMSSW_BASE for var in ( "inputFileList" , "json" , "configTemplate" ) : self . _datasets [ name ] [ var ] = os . path . expandvars ( self . _datasets [ name ] [ var ] ) # Get number of jobs from lines in inputfilelist self . _datasets [ name ] [ "njobs" ] = 0 try : with open ( self . _datasets [ name ] [ "inputFileList" ] , "r" ) as filelist : for line in filelist : if "CastorPool" in line : continue # ignore empty lines if not line . strip ( ) == "" : self . _datasets [ name ] [ "njobs" ] += 1 except IOError : print ( "Inputfilelist" , self . _datasets [ name ] [ "inputFileList" ] , end = ' ' ) print ( "does not exist." ) sys . exit ( 1 ) if self . _datasets [ name ] [ "njobs" ] == 0 : print ( "Number of jobs is 0. There may be a problem with the inputfilelist:" ) print ( self . _datasets [ name ] [ "inputFileList" ] ) sys . exit ( 1 ) # Check if njobs gets overwritten in .ini-file if config [ "config" ] . has_option ( section , "njobs" ) : if config [ "config" ] . getint ( section , "njobs" ) <= self . _datasets [ name ] [ "njobs" ] : self . _datasets [ name ] [ "njobs" ] = config [ "config" ] . getint ( section , "njobs" ) else : print ( "'njobs' is bigger than the number of files for this" , end = ' ' ) print ( "dataset:" , self . _datasets [ name ] [ "njobs" ] ) print ( "Using default." ) else : print ( "No number of jobs specified. Using number of files in" , end = ' ' ) print ( "inputfilelist as the number of jobs." ) # check if local weights override global weights and resolve name clashes for weight_name , weight_values in common_weights . items ( ) : for key , weight in weight_dict . items ( ) : if any ( [ weight_name in w for w in weight ] ) : self . _common_weights [ weight_name + config [ "config" ] . config_path ] = weight_values self . _weight_dict [ key ] = [ mps_tools . replace_factors ( w , weight_name , weight_name + config [ "config" ] . config_path ) for w in weight ] else : self . _common_weights [ weight_name ] = weight_values self . _weight_dict [ key ] = weight os . environ [ "datasetdir" ] = cache_datasetdir if len ( self . _datasets ) == 0 : print ( "No dataset section defined in '{0}'" . format ( ", " . join ( [ self . _args . aligmentConfig ] + self . _external_datasets . keys ( ) ) ) ) print ( "At least one section '[dataset:<name>]' is required." ) sys . exit ( 1 ) self . _global_tag = self . _datasets [ name ] [ "globaltag" ]